2024-11-15 16:50:32,026 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-15 16:50:32,039 main DEBUG Took 0.010256 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 16:50:32,039 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 16:50:32,039 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 16:50:32,040 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 16:50:32,042 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:50:32,048 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 16:50:32,060 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:50:32,061 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:50:32,062 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:50:32,062 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:50:32,063 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:50:32,063 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:50:32,064 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:50:32,064 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:50:32,064 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:50:32,065 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:50:32,065 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:50:32,066 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:50:32,066 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:50:32,066 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:50:32,067 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:50:32,067 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:50:32,068 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:50:32,068 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:50:32,068 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:50:32,069 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:50:32,069 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:50:32,069 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:50:32,070 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:50:32,070 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:50:32,070 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:50:32,071 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 16:50:32,072 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:50:32,073 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 16:50:32,075 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 16:50:32,075 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 16:50:32,076 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 16:50:32,077 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 16:50:32,085 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 16:50:32,088 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 16:50:32,089 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 16:50:32,090 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 16:50:32,090 main DEBUG createAppenders(={Console}) 2024-11-15 16:50:32,091 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-15 16:50:32,091 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-15 16:50:32,091 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-15 16:50:32,092 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 16:50:32,092 main DEBUG OutputStream closed 2024-11-15 16:50:32,092 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 16:50:32,092 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 16:50:32,093 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-15 16:50:32,173 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 16:50:32,176 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 16:50:32,177 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 16:50:32,178 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 16:50:32,179 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 16:50:32,180 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 16:50:32,180 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 16:50:32,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 16:50:32,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 16:50:32,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 16:50:32,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 16:50:32,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 16:50:32,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 16:50:32,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 16:50:32,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 16:50:32,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 16:50:32,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 16:50:32,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 16:50:32,188 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 16:50:32,188 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-15 16:50:32,188 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 16:50:32,189 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-15T16:50:32,460 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9 2024-11-15 16:50:32,463 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 16:50:32,463 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T16:50:32,472 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-15T16:50:32,478 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayValueCompression timeout: 13 mins 2024-11-15T16:50:32,498 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T16:50:32,543 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T16:50:32,543 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T16:50:32,555 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-15T16:50:32,571 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff, deleteOnExit=true 2024-11-15T16:50:32,571 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-15T16:50:32,572 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/test.cache.data in system properties and HBase conf 2024-11-15T16:50:32,573 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/hadoop.tmp.dir in system properties and HBase conf 2024-11-15T16:50:32,573 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/hadoop.log.dir in system properties and HBase conf 2024-11-15T16:50:32,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-15T16:50:32,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-15T16:50:32,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-15T16:50:32,657 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T16:50:32,753 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:50:32,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-15T16:50:32,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-15T16:50:32,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-15T16:50:32,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T16:50:32,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-15T16:50:32,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-15T16:50:32,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T16:50:32,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T16:50:32,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-15T16:50:32,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/nfs.dump.dir in system properties and HBase conf 2024-11-15T16:50:32,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/java.io.tmpdir in system properties and HBase conf 2024-11-15T16:50:32,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T16:50:32,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-15T16:50:32,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-15T16:50:33,588 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-15T16:50:33,668 INFO [Time-limited test {}] log.Log(170): Logging initialized @2373ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-15T16:50:33,756 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T16:50:33,840 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T16:50:33,892 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T16:50:33,892 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T16:50:33,894 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T16:50:33,909 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T16:50:33,912 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/hadoop.log.dir/,AVAILABLE} 2024-11-15T16:50:33,913 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T16:50:34,107 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3717288f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/java.io.tmpdir/jetty-localhost-44069-hadoop-hdfs-3_4_1-tests_jar-_-any-15108720999502035230/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T16:50:34,113 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:44069} 2024-11-15T16:50:34,114 INFO [Time-limited test {}] server.Server(415): Started @2818ms 2024-11-15T16:50:34,518 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T16:50:34,525 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T16:50:34,527 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T16:50:34,527 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T16:50:34,527 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T16:50:34,528 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1f76f489{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/hadoop.log.dir/,AVAILABLE} 2024-11-15T16:50:34,528 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@433df981{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T16:50:34,656 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@36632d60{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/java.io.tmpdir/jetty-localhost-37791-hadoop-hdfs-3_4_1-tests_jar-_-any-3187454333482489112/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T16:50:34,657 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@751d2fa4{HTTP/1.1, (http/1.1)}{localhost:37791} 2024-11-15T16:50:34,657 INFO [Time-limited test {}] server.Server(415): Started @3362ms 2024-11-15T16:50:34,715 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T16:50:34,865 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T16:50:34,870 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T16:50:34,872 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T16:50:34,872 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T16:50:34,872 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T16:50:34,876 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b5fc47c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/hadoop.log.dir/,AVAILABLE} 2024-11-15T16:50:34,876 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ebbf344{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T16:50:34,996 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4546bb60{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/java.io.tmpdir/jetty-localhost-37193-hadoop-hdfs-3_4_1-tests_jar-_-any-11480952562081958251/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T16:50:34,997 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7c7d32f8{HTTP/1.1, (http/1.1)}{localhost:37193} 2024-11-15T16:50:34,998 INFO [Time-limited test {}] server.Server(415): Started @3703ms 2024-11-15T16:50:35,000 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T16:50:35,047 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T16:50:35,053 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T16:50:35,060 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T16:50:35,060 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T16:50:35,060 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T16:50:35,061 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7da22a2e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/hadoop.log.dir/,AVAILABLE} 2024-11-15T16:50:35,062 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f079a76{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T16:50:35,210 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data2/current/BP-1263208986-172.17.0.2-1731689433335/current, will proceed with Du for space computation calculation, 2024-11-15T16:50:35,210 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data4/current/BP-1263208986-172.17.0.2-1731689433335/current, will proceed with Du for space computation calculation, 2024-11-15T16:50:35,210 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data3/current/BP-1263208986-172.17.0.2-1731689433335/current, will proceed with Du for space computation calculation, 2024-11-15T16:50:35,212 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data1/current/BP-1263208986-172.17.0.2-1731689433335/current, will proceed with Du for space computation calculation, 2024-11-15T16:50:35,213 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@43206bef{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/java.io.tmpdir/jetty-localhost-45759-hadoop-hdfs-3_4_1-tests_jar-_-any-9804346082533222760/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T16:50:35,214 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@228ffa29{HTTP/1.1, (http/1.1)}{localhost:45759} 2024-11-15T16:50:35,215 INFO [Time-limited test {}] server.Server(415): Started @3919ms 2024-11-15T16:50:35,217 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T16:50:35,276 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T16:50:35,276 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T16:50:35,361 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd1a78fe98d054c67 with lease ID 0xf3f800dddf9e607c: Processing first storage report for DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3 from datanode DatanodeRegistration(127.0.0.1:39325, datanodeUuid=be47a01f-b238-4003-9f9d-9e18c078acd0, infoPort=40741, infoSecurePort=0, ipcPort=42261, storageInfo=lv=-57;cid=testClusterID;nsid=1274199738;c=1731689433335) 2024-11-15T16:50:35,362 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd1a78fe98d054c67 with lease ID 0xf3f800dddf9e607c: from storage DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3 node DatanodeRegistration(127.0.0.1:39325, datanodeUuid=be47a01f-b238-4003-9f9d-9e18c078acd0, infoPort=40741, infoSecurePort=0, ipcPort=42261, storageInfo=lv=-57;cid=testClusterID;nsid=1274199738;c=1731689433335), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-15T16:50:35,363 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x30f9b556cfdd9769 with lease ID 0xf3f800dddf9e607d: Processing first storage report for DS-8100e98b-4a9e-44f7-9998-a14437efca4b from datanode DatanodeRegistration(127.0.0.1:37733, datanodeUuid=2db290b3-fd4e-44b9-9c0f-0aaa0bec18d5, infoPort=36277, infoSecurePort=0, ipcPort=41293, storageInfo=lv=-57;cid=testClusterID;nsid=1274199738;c=1731689433335) 2024-11-15T16:50:35,363 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x30f9b556cfdd9769 with lease ID 0xf3f800dddf9e607d: from storage DS-8100e98b-4a9e-44f7-9998-a14437efca4b node DatanodeRegistration(127.0.0.1:37733, datanodeUuid=2db290b3-fd4e-44b9-9c0f-0aaa0bec18d5, infoPort=36277, infoSecurePort=0, ipcPort=41293, storageInfo=lv=-57;cid=testClusterID;nsid=1274199738;c=1731689433335), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T16:50:35,364 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x30f9b556cfdd9769 with lease ID 0xf3f800dddf9e607d: Processing first storage report for DS-07e40e7b-163e-4424-9637-6ecabbb1aba9 from datanode DatanodeRegistration(127.0.0.1:37733, datanodeUuid=2db290b3-fd4e-44b9-9c0f-0aaa0bec18d5, infoPort=36277, infoSecurePort=0, ipcPort=41293, storageInfo=lv=-57;cid=testClusterID;nsid=1274199738;c=1731689433335) 2024-11-15T16:50:35,364 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x30f9b556cfdd9769 with lease ID 0xf3f800dddf9e607d: from storage DS-07e40e7b-163e-4424-9637-6ecabbb1aba9 node DatanodeRegistration(127.0.0.1:37733, datanodeUuid=2db290b3-fd4e-44b9-9c0f-0aaa0bec18d5, infoPort=36277, infoSecurePort=0, ipcPort=41293, storageInfo=lv=-57;cid=testClusterID;nsid=1274199738;c=1731689433335), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T16:50:35,364 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd1a78fe98d054c67 with lease ID 0xf3f800dddf9e607c: Processing first storage report for DS-8070926f-908f-46c7-b568-d1456d8ed03d from datanode DatanodeRegistration(127.0.0.1:39325, datanodeUuid=be47a01f-b238-4003-9f9d-9e18c078acd0, infoPort=40741, infoSecurePort=0, ipcPort=42261, storageInfo=lv=-57;cid=testClusterID;nsid=1274199738;c=1731689433335) 2024-11-15T16:50:35,364 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd1a78fe98d054c67 with lease ID 0xf3f800dddf9e607c: from storage DS-8070926f-908f-46c7-b568-d1456d8ed03d node DatanodeRegistration(127.0.0.1:39325, datanodeUuid=be47a01f-b238-4003-9f9d-9e18c078acd0, infoPort=40741, infoSecurePort=0, ipcPort=42261, storageInfo=lv=-57;cid=testClusterID;nsid=1274199738;c=1731689433335), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T16:50:35,388 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data5/current/BP-1263208986-172.17.0.2-1731689433335/current, will proceed with Du for space computation calculation, 2024-11-15T16:50:35,390 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data6/current/BP-1263208986-172.17.0.2-1731689433335/current, will proceed with Du for space computation calculation, 2024-11-15T16:50:35,417 WARN [Thread-123 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T16:50:35,423 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1f3dd8be1c49f127 with lease ID 0xf3f800dddf9e607e: Processing first storage report for DS-4629d782-bcf0-4860-8f68-d4ca2009a650 from datanode DatanodeRegistration(127.0.0.1:42975, datanodeUuid=e2e4cabe-ae6c-4cad-a7b2-f853e118a4ff, infoPort=37569, infoSecurePort=0, ipcPort=44899, storageInfo=lv=-57;cid=testClusterID;nsid=1274199738;c=1731689433335) 2024-11-15T16:50:35,423 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1f3dd8be1c49f127 with lease ID 0xf3f800dddf9e607e: from storage DS-4629d782-bcf0-4860-8f68-d4ca2009a650 node DatanodeRegistration(127.0.0.1:42975, datanodeUuid=e2e4cabe-ae6c-4cad-a7b2-f853e118a4ff, infoPort=37569, infoSecurePort=0, ipcPort=44899, storageInfo=lv=-57;cid=testClusterID;nsid=1274199738;c=1731689433335), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T16:50:35,423 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1f3dd8be1c49f127 with lease ID 0xf3f800dddf9e607e: Processing first storage report for DS-d582ec6f-818f-4a34-8585-fad0bae7817b from datanode DatanodeRegistration(127.0.0.1:42975, datanodeUuid=e2e4cabe-ae6c-4cad-a7b2-f853e118a4ff, infoPort=37569, infoSecurePort=0, ipcPort=44899, storageInfo=lv=-57;cid=testClusterID;nsid=1274199738;c=1731689433335) 2024-11-15T16:50:35,424 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1f3dd8be1c49f127 with lease ID 0xf3f800dddf9e607e: from storage DS-d582ec6f-818f-4a34-8585-fad0bae7817b node DatanodeRegistration(127.0.0.1:42975, datanodeUuid=e2e4cabe-ae6c-4cad-a7b2-f853e118a4ff, infoPort=37569, infoSecurePort=0, ipcPort=44899, storageInfo=lv=-57;cid=testClusterID;nsid=1274199738;c=1731689433335), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T16:50:35,593 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9 2024-11-15T16:50:35,685 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/zookeeper_0, clientPort=50974, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T16:50:35,694 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50974 2024-11-15T16:50:35,707 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:50:35,710 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:50:35,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741825_1001 (size=7) 2024-11-15T16:50:35,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741825_1001 (size=7) 2024-11-15T16:50:35,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741825_1001 (size=7) 2024-11-15T16:50:36,324 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd with version=8 2024-11-15T16:50:36,324 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/hbase-staging 2024-11-15T16:50:36,646 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a9bde5b3a1b5:0 server-side Connection retries=45 2024-11-15T16:50:36,658 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:50:36,658 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T16:50:36,663 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T16:50:36,663 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:50:36,663 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T16:50:36,803 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T16:50:36,865 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T16:50:36,874 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T16:50:36,878 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T16:50:36,906 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 26964 (auto-detected) 2024-11-15T16:50:36,907 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-15T16:50:36,926 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43823 2024-11-15T16:50:36,948 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43823 connecting to ZooKeeper ensemble=127.0.0.1:50974 2024-11-15T16:50:36,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:438230x0, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T16:50:36,983 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43823-0x101acd7477a0000 connected 2024-11-15T16:50:37,010 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:50:37,014 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:50:37,024 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:50:37,028 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd, hbase.cluster.distributed=false 2024-11-15T16:50:37,051 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T16:50:37,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43823 2024-11-15T16:50:37,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43823 2024-11-15T16:50:37,057 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43823 2024-11-15T16:50:37,057 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43823 2024-11-15T16:50:37,057 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43823 2024-11-15T16:50:37,170 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a9bde5b3a1b5:0 server-side Connection retries=45 2024-11-15T16:50:37,172 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:50:37,172 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T16:50:37,172 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T16:50:37,173 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:50:37,173 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T16:50:37,176 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T16:50:37,178 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T16:50:37,179 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38139 2024-11-15T16:50:37,181 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38139 connecting to ZooKeeper ensemble=127.0.0.1:50974 2024-11-15T16:50:37,182 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:50:37,184 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:50:37,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:381390x0, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T16:50:37,192 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:50:37,192 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38139-0x101acd7477a0001 connected 2024-11-15T16:50:37,196 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T16:50:37,204 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T16:50:37,207 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T16:50:37,211 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T16:50:37,212 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38139 2024-11-15T16:50:37,212 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38139 2024-11-15T16:50:37,213 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38139 2024-11-15T16:50:37,213 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38139 2024-11-15T16:50:37,214 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38139 2024-11-15T16:50:37,231 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a9bde5b3a1b5:0 server-side Connection retries=45 2024-11-15T16:50:37,231 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:50:37,231 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T16:50:37,232 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T16:50:37,232 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:50:37,232 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T16:50:37,232 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T16:50:37,232 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T16:50:37,233 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45441 2024-11-15T16:50:37,235 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45441 connecting to ZooKeeper ensemble=127.0.0.1:50974 2024-11-15T16:50:37,236 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:50:37,238 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:50:37,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:454410x0, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T16:50:37,244 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45441-0x101acd7477a0002 connected 2024-11-15T16:50:37,244 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:50:37,244 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T16:50:37,246 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T16:50:37,246 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T16:50:37,248 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T16:50:37,249 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45441 2024-11-15T16:50:37,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45441 2024-11-15T16:50:37,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45441 2024-11-15T16:50:37,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45441 2024-11-15T16:50:37,252 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45441 2024-11-15T16:50:37,268 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a9bde5b3a1b5:0 server-side Connection retries=45 2024-11-15T16:50:37,268 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:50:37,268 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T16:50:37,268 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T16:50:37,268 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:50:37,268 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T16:50:37,269 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T16:50:37,269 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T16:50:37,271 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40715 2024-11-15T16:50:37,272 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40715 connecting to ZooKeeper ensemble=127.0.0.1:50974 2024-11-15T16:50:37,274 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:50:37,275 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:50:37,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:407150x0, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T16:50:37,281 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40715-0x101acd7477a0003 connected 2024-11-15T16:50:37,281 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:50:37,282 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T16:50:37,283 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T16:50:37,284 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T16:50:37,286 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T16:50:37,286 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40715 2024-11-15T16:50:37,287 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40715 2024-11-15T16:50:37,288 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40715 2024-11-15T16:50:37,291 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40715 2024-11-15T16:50:37,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40715 2024-11-15T16:50:37,308 DEBUG [M:0;a9bde5b3a1b5:43823 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a9bde5b3a1b5:43823 2024-11-15T16:50:37,309 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a9bde5b3a1b5,43823,1731689436438 2024-11-15T16:50:37,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:50:37,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:50:37,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:50:37,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:50:37,318 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a9bde5b3a1b5,43823,1731689436438 2024-11-15T16:50:37,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T16:50:37,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T16:50:37,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T16:50:37,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:37,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:37,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:37,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:37,340 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T16:50:37,341 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a9bde5b3a1b5,43823,1731689436438 from backup master directory 2024-11-15T16:50:37,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a9bde5b3a1b5,43823,1731689436438 2024-11-15T16:50:37,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:50:37,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:50:37,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:50:37,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:50:37,346 WARN [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T16:50:37,346 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a9bde5b3a1b5,43823,1731689436438 2024-11-15T16:50:37,348 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T16:50:37,350 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T16:50:37,407 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/hbase.id] with ID: e3801ad5-a8ea-4892-a4a8-717494d68971 2024-11-15T16:50:37,407 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/.tmp/hbase.id 2024-11-15T16:50:37,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741826_1002 (size=42) 2024-11-15T16:50:37,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741826_1002 (size=42) 2024-11-15T16:50:37,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741826_1002 (size=42) 2024-11-15T16:50:37,424 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/.tmp/hbase.id]:[hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/hbase.id] 2024-11-15T16:50:37,468 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:50:37,472 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T16:50:37,491 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-15T16:50:37,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:37,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:37,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:37,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:37,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741827_1003 (size=196) 2024-11-15T16:50:37,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741827_1003 (size=196) 2024-11-15T16:50:37,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741827_1003 (size=196) 2024-11-15T16:50:37,532 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T16:50:37,534 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T16:50:37,541 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:37,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741828_1004 (size=1189) 2024-11-15T16:50:37,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741828_1004 (size=1189) 2024-11-15T16:50:37,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741828_1004 (size=1189) 2024-11-15T16:50:37,594 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store 2024-11-15T16:50:37,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741829_1005 (size=34) 2024-11-15T16:50:37,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741829_1005 (size=34) 2024-11-15T16:50:37,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741829_1005 (size=34) 2024-11-15T16:50:37,621 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T16:50:37,626 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:37,627 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T16:50:37,628 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:50:37,628 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:50:37,630 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T16:50:37,630 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:50:37,631 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:50:37,632 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731689437627Disabling compacts and flushes for region at 1731689437627Disabling writes for close at 1731689437630 (+3 ms)Writing region close event to WAL at 1731689437630Closed at 1731689437630 2024-11-15T16:50:37,635 WARN [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/.initializing 2024-11-15T16:50:37,635 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/WALs/a9bde5b3a1b5,43823,1731689436438 2024-11-15T16:50:37,646 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:50:37,663 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a9bde5b3a1b5%2C43823%2C1731689436438, suffix=, logDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/WALs/a9bde5b3a1b5,43823,1731689436438, archiveDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/oldWALs, maxLogs=10 2024-11-15T16:50:37,705 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/WALs/a9bde5b3a1b5,43823,1731689436438/a9bde5b3a1b5%2C43823%2C1731689436438.1731689437670, exclude list is [], retry=0 2024-11-15T16:50:37,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:50:37,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:37,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:37,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:37,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-15T16:50:37,770 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/WALs/a9bde5b3a1b5,43823,1731689436438/a9bde5b3a1b5%2C43823%2C1731689436438.1731689437670 2024-11-15T16:50:37,770 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:36277:36277)] 2024-11-15T16:50:37,771 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:37,771 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:37,775 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:50:37,775 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:50:37,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:50:37,837 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T16:50:37,840 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:37,843 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:50:37,843 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:50:37,846 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T16:50:37,846 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:37,847 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:37,847 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:50:37,850 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T16:50:37,850 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:37,851 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:37,851 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:50:37,854 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T16:50:37,854 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:37,855 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:37,855 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:50:37,858 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:50:37,859 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:50:37,864 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:50:37,864 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:50:37,867 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T16:50:37,870 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:50:37,875 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:50:37,876 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64185159, jitterRate=-0.04356659948825836}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T16:50:37,883 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731689437787Initializing all the Stores at 1731689437789 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689437790 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689437790Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689437791 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689437791Cleaning up temporary data from old regions at 1731689437864 (+73 ms)Region opened successfully at 1731689437883 (+19 ms) 2024-11-15T16:50:37,884 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T16:50:37,927 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4625cea3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a9bde5b3a1b5/172.17.0.2:0 2024-11-15T16:50:37,962 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T16:50:37,975 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-15T16:50:37,975 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-15T16:50:37,979 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T16:50:37,981 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T16:50:37,986 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T16:50:37,986 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T16:50:38,017 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T16:50:38,025 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T16:50:38,028 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T16:50:38,030 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T16:50:38,032 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T16:50:38,034 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T16:50:38,036 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T16:50:38,039 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T16:50:38,042 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T16:50:38,044 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T16:50:38,046 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T16:50:38,070 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T16:50:38,072 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T16:50:38,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T16:50:38,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T16:50:38,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T16:50:38,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:38,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:38,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:38,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T16:50:38,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:38,086 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a9bde5b3a1b5,43823,1731689436438, sessionid=0x101acd7477a0000, setting cluster-up flag (Was=false) 2024-11-15T16:50:38,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:38,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:38,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:38,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:38,113 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T16:50:38,115 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a9bde5b3a1b5,43823,1731689436438 2024-11-15T16:50:38,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:38,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:38,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:38,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:38,130 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T16:50:38,133 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a9bde5b3a1b5,43823,1731689436438 2024-11-15T16:50:38,139 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T16:50:38,197 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(746): ClusterId : e3801ad5-a8ea-4892-a4a8-717494d68971 2024-11-15T16:50:38,197 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(746): ClusterId : e3801ad5-a8ea-4892-a4a8-717494d68971 2024-11-15T16:50:38,197 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(746): ClusterId : e3801ad5-a8ea-4892-a4a8-717494d68971 2024-11-15T16:50:38,199 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T16:50:38,199 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T16:50:38,199 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T16:50:38,205 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T16:50:38,205 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T16:50:38,205 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T16:50:38,205 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T16:50:38,205 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T16:50:38,205 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T16:50:38,209 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T16:50:38,209 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T16:50:38,210 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c762f86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a9bde5b3a1b5/172.17.0.2:0 2024-11-15T16:50:38,210 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6866c80f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a9bde5b3a1b5/172.17.0.2:0 2024-11-15T16:50:38,210 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T16:50:38,211 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a9de967, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a9bde5b3a1b5/172.17.0.2:0 2024-11-15T16:50:38,227 INFO [AsyncFSWAL-0-hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData-prefix:a9bde5b3a1b5,43823,1731689436438 {}] compress.Compression(560): Loaded codec org.apache.hadoop.hbase.io.compress.ReusableStreamGzipCodec for compression algorithm GZ 2024-11-15T16:50:38,232 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;a9bde5b3a1b5:40715 2024-11-15T16:50:38,232 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a9bde5b3a1b5:38139 2024-11-15T16:50:38,232 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a9bde5b3a1b5:45441 2024-11-15T16:50:38,236 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T16:50:38,236 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T16:50:38,236 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T16:50:38,236 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T16:50:38,236 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T16:50:38,236 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T16:50:38,236 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T16:50:38,237 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T16:50:38,237 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T16:50:38,240 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(2659): reportForDuty to master=a9bde5b3a1b5,43823,1731689436438 with port=40715, startcode=1731689437267 2024-11-15T16:50:38,240 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(2659): reportForDuty to master=a9bde5b3a1b5,43823,1731689436438 with port=38139, startcode=1731689437129 2024-11-15T16:50:38,240 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(2659): reportForDuty to master=a9bde5b3a1b5,43823,1731689436438 with port=45441, startcode=1731689437230 2024-11-15T16:50:38,247 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T16:50:38,254 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T16:50:38,254 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T16:50:38,254 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T16:50:38,257 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T16:50:38,264 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T16:50:38,270 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a9bde5b3a1b5,43823,1731689436438 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T16:50:38,279 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a9bde5b3a1b5:0, corePoolSize=5, maxPoolSize=5 2024-11-15T16:50:38,280 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a9bde5b3a1b5:0, corePoolSize=5, maxPoolSize=5 2024-11-15T16:50:38,280 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=5, maxPoolSize=5 2024-11-15T16:50:38,280 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=5, maxPoolSize=5 2024-11-15T16:50:38,280 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a9bde5b3a1b5:0, corePoolSize=10, maxPoolSize=10 2024-11-15T16:50:38,281 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,281 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=2, maxPoolSize=2 2024-11-15T16:50:38,281 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,290 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T16:50:38,290 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731689468290 2024-11-15T16:50:38,290 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T16:50:38,292 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T16:50:38,293 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T16:50:38,297 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39411, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T16:50:38,297 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39775, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T16:50:38,297 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57525, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T16:50:38,298 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T16:50:38,298 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T16:50:38,299 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T16:50:38,299 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T16:50:38,300 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:38,300 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T16:50:38,300 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,304 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T16:50:38,304 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43823 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:50:38,306 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T16:50:38,306 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T16:50:38,310 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T16:50:38,311 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T16:50:38,311 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43823 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:50:38,312 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43823 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:50:38,313 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.large.0-1731689438312,5,FailOnTimeoutGroup] 2024-11-15T16:50:38,314 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.small.0-1731689438313,5,FailOnTimeoutGroup] 2024-11-15T16:50:38,314 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,314 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T16:50:38,316 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,316 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741831_1007 (size=1321) 2024-11-15T16:50:38,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741831_1007 (size=1321) 2024-11-15T16:50:38,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741831_1007 (size=1321) 2024-11-15T16:50:38,358 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T16:50:38,359 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd 2024-11-15T16:50:38,366 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-15T16:50:38,366 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-15T16:50:38,366 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-15T16:50:38,366 WARN [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-15T16:50:38,366 WARN [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-15T16:50:38,366 WARN [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-15T16:50:38,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741832_1008 (size=32) 2024-11-15T16:50:38,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741832_1008 (size=32) 2024-11-15T16:50:38,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741832_1008 (size=32) 2024-11-15T16:50:38,375 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:38,377 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T16:50:38,379 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T16:50:38,380 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:38,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:50:38,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T16:50:38,383 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T16:50:38,383 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:38,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:50:38,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T16:50:38,387 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T16:50:38,387 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:38,388 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:50:38,388 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T16:50:38,390 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T16:50:38,391 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:38,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:50:38,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T16:50:38,393 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740 2024-11-15T16:50:38,394 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740 2024-11-15T16:50:38,396 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T16:50:38,396 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T16:50:38,397 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T16:50:38,399 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T16:50:38,404 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:50:38,405 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69523759, jitterRate=0.035984739661216736}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T16:50:38,409 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731689438375Initializing all the Stores at 1731689438377 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689438377Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689438377Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689438377Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689438377Cleaning up temporary data from old regions at 1731689438396 (+19 ms)Region opened successfully at 1731689438409 (+13 ms) 2024-11-15T16:50:38,409 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T16:50:38,409 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T16:50:38,409 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T16:50:38,409 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T16:50:38,409 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T16:50:38,411 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T16:50:38,411 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731689438409Disabling compacts and flushes for region at 1731689438409Disabling writes for close at 1731689438409Writing region close event to WAL at 1731689438410 (+1 ms)Closed at 1731689438411 (+1 ms) 2024-11-15T16:50:38,415 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T16:50:38,415 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T16:50:38,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T16:50:38,433 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T16:50:38,436 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T16:50:38,467 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(2659): reportForDuty to master=a9bde5b3a1b5,43823,1731689436438 with port=40715, startcode=1731689437267 2024-11-15T16:50:38,467 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(2659): reportForDuty to master=a9bde5b3a1b5,43823,1731689436438 with port=45441, startcode=1731689437230 2024-11-15T16:50:38,467 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(2659): reportForDuty to master=a9bde5b3a1b5,43823,1731689436438 with port=38139, startcode=1731689437129 2024-11-15T16:50:38,470 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43823 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a9bde5b3a1b5,40715,1731689437267 2024-11-15T16:50:38,473 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43823 {}] master.ServerManager(517): Registering regionserver=a9bde5b3a1b5,40715,1731689437267 2024-11-15T16:50:38,481 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43823 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a9bde5b3a1b5,45441,1731689437230 2024-11-15T16:50:38,481 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43823 {}] master.ServerManager(517): Registering regionserver=a9bde5b3a1b5,45441,1731689437230 2024-11-15T16:50:38,481 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd 2024-11-15T16:50:38,481 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40387 2024-11-15T16:50:38,481 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T16:50:38,484 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43823 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a9bde5b3a1b5,38139,1731689437129 2024-11-15T16:50:38,484 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd 2024-11-15T16:50:38,484 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40387 2024-11-15T16:50:38,484 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43823 {}] master.ServerManager(517): Registering regionserver=a9bde5b3a1b5,38139,1731689437129 2024-11-15T16:50:38,485 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T16:50:38,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T16:50:38,489 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd 2024-11-15T16:50:38,489 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40387 2024-11-15T16:50:38,489 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T16:50:38,493 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] zookeeper.ZKUtil(111): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a9bde5b3a1b5,40715,1731689437267 2024-11-15T16:50:38,493 WARN [RS:2;a9bde5b3a1b5:40715 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T16:50:38,493 INFO [RS:2;a9bde5b3a1b5:40715 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:38,493 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,40715,1731689437267 2024-11-15T16:50:38,494 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a9bde5b3a1b5,40715,1731689437267] 2024-11-15T16:50:38,494 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a9bde5b3a1b5,45441,1731689437230] 2024-11-15T16:50:38,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T16:50:38,495 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] zookeeper.ZKUtil(111): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a9bde5b3a1b5,45441,1731689437230 2024-11-15T16:50:38,495 WARN [RS:1;a9bde5b3a1b5:45441 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T16:50:38,495 INFO [RS:1;a9bde5b3a1b5:45441 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:38,495 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] zookeeper.ZKUtil(111): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a9bde5b3a1b5,38139,1731689437129 2024-11-15T16:50:38,495 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,45441,1731689437230 2024-11-15T16:50:38,495 WARN [RS:0;a9bde5b3a1b5:38139 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T16:50:38,495 INFO [RS:0;a9bde5b3a1b5:38139 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:38,496 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129 2024-11-15T16:50:38,496 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a9bde5b3a1b5,38139,1731689437129] 2024-11-15T16:50:38,523 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T16:50:38,523 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T16:50:38,523 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T16:50:38,537 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T16:50:38,537 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T16:50:38,537 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T16:50:38,543 INFO [RS:2;a9bde5b3a1b5:40715 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T16:50:38,543 INFO [RS:1;a9bde5b3a1b5:45441 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T16:50:38,543 INFO [RS:0;a9bde5b3a1b5:38139 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T16:50:38,543 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,543 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,543 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,544 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T16:50:38,544 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T16:50:38,544 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T16:50:38,550 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T16:50:38,550 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T16:50:38,550 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T16:50:38,552 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,552 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,552 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,552 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,552 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,552 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,552 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,552 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,552 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,552 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,552 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,552 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,552 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,552 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,552 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,553 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=2, maxPoolSize=2 2024-11-15T16:50:38,553 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,553 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,553 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,553 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,553 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,553 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=2, maxPoolSize=2 2024-11-15T16:50:38,553 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,553 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=2, maxPoolSize=2 2024-11-15T16:50:38,553 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,553 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,553 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,553 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,553 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,553 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,553 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:50:38,553 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,553 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,554 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:50:38,554 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,554 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,554 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,554 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,554 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,554 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,554 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:50:38,554 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:50:38,554 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,554 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:50:38,555 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,555 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,555 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,555 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:50:38,555 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,555 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,45441,1731689437230-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T16:50:38,555 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,555 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,555 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:50:38,556 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,556 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,556 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,556 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,40715,1731689437267-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T16:50:38,562 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,562 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,562 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,562 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,562 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,562 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,38139,1731689437129-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T16:50:38,578 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T16:50:38,580 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,40715,1731689437267-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,580 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,580 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.Replication(171): a9bde5b3a1b5,40715,1731689437267 started 2024-11-15T16:50:38,580 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T16:50:38,581 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,45441,1731689437230-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,581 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,581 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.Replication(171): a9bde5b3a1b5,45441,1731689437230 started 2024-11-15T16:50:38,586 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T16:50:38,587 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,38139,1731689437129-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,587 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,587 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.Replication(171): a9bde5b3a1b5,38139,1731689437129 started 2024-11-15T16:50:38,587 WARN [a9bde5b3a1b5:43823 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-15T16:50:38,598 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,599 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(1482): Serving as a9bde5b3a1b5,40715,1731689437267, RpcServer on a9bde5b3a1b5/172.17.0.2:40715, sessionid=0x101acd7477a0003 2024-11-15T16:50:38,599 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T16:50:38,599 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a9bde5b3a1b5,40715,1731689437267 2024-11-15T16:50:38,600 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,40715,1731689437267' 2024-11-15T16:50:38,600 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T16:50:38,601 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T16:50:38,601 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T16:50:38,601 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T16:50:38,602 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a9bde5b3a1b5,40715,1731689437267 2024-11-15T16:50:38,602 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,40715,1731689437267' 2024-11-15T16:50:38,602 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T16:50:38,602 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T16:50:38,603 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T16:50:38,603 INFO [RS:2;a9bde5b3a1b5:40715 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T16:50:38,603 INFO [RS:2;a9bde5b3a1b5:40715 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T16:50:38,605 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,605 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(1482): Serving as a9bde5b3a1b5,45441,1731689437230, RpcServer on a9bde5b3a1b5/172.17.0.2:45441, sessionid=0x101acd7477a0002 2024-11-15T16:50:38,605 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T16:50:38,605 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a9bde5b3a1b5,45441,1731689437230 2024-11-15T16:50:38,605 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,45441,1731689437230' 2024-11-15T16:50:38,605 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T16:50:38,606 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T16:50:38,607 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T16:50:38,607 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T16:50:38,607 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a9bde5b3a1b5,45441,1731689437230 2024-11-15T16:50:38,607 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,45441,1731689437230' 2024-11-15T16:50:38,607 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T16:50:38,608 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T16:50:38,608 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T16:50:38,608 INFO [RS:1;a9bde5b3a1b5:45441 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T16:50:38,608 INFO [RS:1;a9bde5b3a1b5:45441 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T16:50:38,611 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:38,611 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(1482): Serving as a9bde5b3a1b5,38139,1731689437129, RpcServer on a9bde5b3a1b5/172.17.0.2:38139, sessionid=0x101acd7477a0001 2024-11-15T16:50:38,611 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T16:50:38,611 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a9bde5b3a1b5,38139,1731689437129 2024-11-15T16:50:38,611 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,38139,1731689437129' 2024-11-15T16:50:38,612 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T16:50:38,612 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T16:50:38,613 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T16:50:38,613 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T16:50:38,614 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a9bde5b3a1b5,38139,1731689437129 2024-11-15T16:50:38,614 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,38139,1731689437129' 2024-11-15T16:50:38,614 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T16:50:38,614 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T16:50:38,615 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T16:50:38,615 INFO [RS:0;a9bde5b3a1b5:38139 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T16:50:38,615 INFO [RS:0;a9bde5b3a1b5:38139 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T16:50:38,709 INFO [RS:2;a9bde5b3a1b5:40715 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:50:38,709 INFO [RS:1;a9bde5b3a1b5:45441 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:50:38,712 INFO [RS:2;a9bde5b3a1b5:40715 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a9bde5b3a1b5%2C40715%2C1731689437267, suffix=, logDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,40715,1731689437267, archiveDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/oldWALs, maxLogs=32 2024-11-15T16:50:38,712 INFO [RS:1;a9bde5b3a1b5:45441 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a9bde5b3a1b5%2C45441%2C1731689437230, suffix=, logDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,45441,1731689437230, archiveDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/oldWALs, maxLogs=32 2024-11-15T16:50:38,716 INFO [RS:0;a9bde5b3a1b5:38139 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:50:38,719 INFO [RS:0;a9bde5b3a1b5:38139 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a9bde5b3a1b5%2C38139%2C1731689437129, suffix=, logDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129, archiveDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/oldWALs, maxLogs=32 2024-11-15T16:50:38,732 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,45441,1731689437230/a9bde5b3a1b5%2C45441%2C1731689437230.1731689438717, exclude list is [], retry=0 2024-11-15T16:50:38,736 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129/a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721, exclude list is [], retry=0 2024-11-15T16:50:38,736 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,40715,1731689437267/a9bde5b3a1b5%2C40715%2C1731689437267.1731689438717, exclude list is [], retry=0 2024-11-15T16:50:38,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:38,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:38,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:38,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:38,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:38,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:38,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:38,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:38,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:38,754 INFO [RS:1;a9bde5b3a1b5:45441 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,45441,1731689437230/a9bde5b3a1b5%2C45441%2C1731689437230.1731689438717 2024-11-15T16:50:38,754 INFO [RS:2;a9bde5b3a1b5:40715 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,40715,1731689437267/a9bde5b3a1b5%2C40715%2C1731689437267.1731689438717 2024-11-15T16:50:38,755 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:40741:40741)] 2024-11-15T16:50:38,755 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:40741:40741)] 2024-11-15T16:50:38,755 INFO [RS:0;a9bde5b3a1b5:38139 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129/a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721 2024-11-15T16:50:38,756 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:36277:36277)] 2024-11-15T16:50:38,839 DEBUG [a9bde5b3a1b5:43823 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-15T16:50:38,847 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(204): Hosts are {a9bde5b3a1b5=0} racks are {/default-rack=0} 2024-11-15T16:50:38,854 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T16:50:38,854 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T16:50:38,854 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-15T16:50:38,854 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T16:50:38,854 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T16:50:38,854 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-15T16:50:38,854 INFO [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T16:50:38,854 INFO [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T16:50:38,854 INFO [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-15T16:50:38,854 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T16:50:38,861 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a9bde5b3a1b5,38139,1731689437129 2024-11-15T16:50:38,866 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a9bde5b3a1b5,38139,1731689437129, state=OPENING 2024-11-15T16:50:38,872 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T16:50:38,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:38,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:38,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:38,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:50:38,875 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:50:38,875 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:50:38,875 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:50:38,875 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:50:38,877 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T16:50:38,879 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,38139,1731689437129}] 2024-11-15T16:50:39,054 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T16:50:39,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39145, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T16:50:39,073 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T16:50:39,074 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:39,074 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T16:50:39,078 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a9bde5b3a1b5%2C38139%2C1731689437129.meta, suffix=.meta, logDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129, archiveDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/oldWALs, maxLogs=32 2024-11-15T16:50:39,096 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129/a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta, exclude list is [], retry=0 2024-11-15T16:50:39,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:39,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:39,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:39,104 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129/a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta 2024-11-15T16:50:39,105 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:36277:36277)] 2024-11-15T16:50:39,105 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:39,108 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T16:50:39,112 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T16:50:39,119 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T16:50:39,125 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T16:50:39,125 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:39,126 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T16:50:39,126 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T16:50:39,130 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T16:50:39,131 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T16:50:39,131 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:39,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:50:39,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T16:50:39,134 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T16:50:39,134 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:39,135 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:50:39,135 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T16:50:39,136 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T16:50:39,137 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:39,138 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:50:39,138 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T16:50:39,139 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T16:50:39,140 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:39,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:50:39,141 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T16:50:39,142 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740 2024-11-15T16:50:39,145 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740 2024-11-15T16:50:39,148 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T16:50:39,148 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T16:50:39,149 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T16:50:39,151 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T16:50:39,153 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63614590, jitterRate=-0.052068740129470825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T16:50:39,153 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T16:50:39,154 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731689439126Writing region info on filesystem at 1731689439127 (+1 ms)Initializing all the Stores at 1731689439129 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689439129Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689439129Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689439130 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689439130Cleaning up temporary data from old regions at 1731689439148 (+18 ms)Running coprocessor post-open hooks at 1731689439153 (+5 ms)Region opened successfully at 1731689439154 (+1 ms) 2024-11-15T16:50:39,162 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731689439045 2024-11-15T16:50:39,174 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T16:50:39,175 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T16:50:39,176 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a9bde5b3a1b5,38139,1731689437129 2024-11-15T16:50:39,178 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a9bde5b3a1b5,38139,1731689437129, state=OPEN 2024-11-15T16:50:39,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:50:39,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:50:39,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:50:39,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:50:39,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:50:39,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:50:39,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:50:39,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:50:39,182 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,38139,1731689437129 2024-11-15T16:50:39,190 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T16:50:39,190 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,38139,1731689437129 in 303 msec 2024-11-15T16:50:39,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T16:50:39,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 768 msec 2024-11-15T16:50:39,200 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T16:50:39,200 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T16:50:39,246 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:50:39,247 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1] 2024-11-15T16:50:39,273 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:50:39,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50975, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:50:39,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1270 sec 2024-11-15T16:50:39,297 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731689439297, completionTime=-1 2024-11-15T16:50:39,300 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-15T16:50:39,301 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T16:50:39,328 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-15T16:50:39,328 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731689499328 2024-11-15T16:50:39,328 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731689559328 2024-11-15T16:50:39,328 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-15T16:50:39,330 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-15T16:50:39,337 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,43823,1731689436438-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:39,337 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,43823,1731689436438-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:39,337 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,43823,1731689436438-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:39,339 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a9bde5b3a1b5:43823, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:39,339 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:39,340 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:39,347 DEBUG [master/a9bde5b3a1b5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T16:50:39,376 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.030sec 2024-11-15T16:50:39,377 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T16:50:39,378 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T16:50:39,379 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T16:50:39,380 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T16:50:39,380 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T16:50:39,381 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,43823,1731689436438-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T16:50:39,381 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,43823,1731689436438-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T16:50:39,386 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T16:50:39,386 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T16:50:39,387 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,43823,1731689436438-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:39,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a940a79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:50:39,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,43823,-1 for getting cluster id 2024-11-15T16:50:39,411 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:50:39,418 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e3801ad5-a8ea-4892-a4a8-717494d68971' 2024-11-15T16:50:39,421 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:50:39,421 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e3801ad5-a8ea-4892-a4a8-717494d68971" 2024-11-15T16:50:39,423 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1108f2f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:50:39,423 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,43823,-1] 2024-11-15T16:50:39,426 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:50:39,427 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:50:39,429 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:50:39,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604ede98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:50:39,432 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:50:39,439 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1] 2024-11-15T16:50:39,439 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:50:39,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:50:39,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a9bde5b3a1b5,43823,1731689436438 2024-11-15T16:50:39,462 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:40387/hbase 2024-11-15T16:50:39,479 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=367, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=260, ProcessCount=11, AvailableMemoryMB=5651 2024-11-15T16:50:39,502 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:39,505 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:39,506 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:50:39,511 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-18595244, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/hregion-18595244, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:39,535 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-18595244/hregion-18595244.1731689439513, exclude list is [], retry=0 2024-11-15T16:50:39,540 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:39,540 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:39,541 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:39,545 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-18595244/hregion-18595244.1731689439513 2024-11-15T16:50:39,545 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:40741:40741)] 2024-11-15T16:50:39,545 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => be59cfdc61180d35f564c8db0b26b902, NAME => 'testReplayEditsWrittenIntoWAL,,1731689439503.be59cfdc61180d35f564c8db0b26b902.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40387/hbase 2024-11-15T16:50:39,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741838_1014 (size=64) 2024-11-15T16:50:39,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741838_1014 (size=64) 2024-11-15T16:50:39,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741838_1014 (size=64) 2024-11-15T16:50:39,561 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731689439503.be59cfdc61180d35f564c8db0b26b902.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:39,563 INFO [StoreOpener-be59cfdc61180d35f564c8db0b26b902-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region be59cfdc61180d35f564c8db0b26b902 2024-11-15T16:50:39,566 INFO [StoreOpener-be59cfdc61180d35f564c8db0b26b902-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be59cfdc61180d35f564c8db0b26b902 columnFamilyName a 2024-11-15T16:50:39,566 DEBUG [StoreOpener-be59cfdc61180d35f564c8db0b26b902-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:39,567 INFO [StoreOpener-be59cfdc61180d35f564c8db0b26b902-1 {}] regionserver.HStore(327): Store=be59cfdc61180d35f564c8db0b26b902/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:39,567 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for be59cfdc61180d35f564c8db0b26b902 2024-11-15T16:50:39,569 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902 2024-11-15T16:50:39,570 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902 2024-11-15T16:50:39,570 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for be59cfdc61180d35f564c8db0b26b902 2024-11-15T16:50:39,570 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for be59cfdc61180d35f564c8db0b26b902 2024-11-15T16:50:39,574 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for be59cfdc61180d35f564c8db0b26b902 2024-11-15T16:50:39,578 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:50:39,579 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened be59cfdc61180d35f564c8db0b26b902; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67014687, jitterRate=-0.0014033466577529907}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T16:50:39,581 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for be59cfdc61180d35f564c8db0b26b902: Writing region info on filesystem at 1731689439561Initializing all the Stores at 1731689439563 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689439563Cleaning up temporary data from old regions at 1731689439570 (+7 ms)Region opened successfully at 1731689439581 (+11 ms) 2024-11-15T16:50:39,581 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing be59cfdc61180d35f564c8db0b26b902, disabling compactions & flushes 2024-11-15T16:50:39,581 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731689439503.be59cfdc61180d35f564c8db0b26b902. 2024-11-15T16:50:39,581 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731689439503.be59cfdc61180d35f564c8db0b26b902. 2024-11-15T16:50:39,581 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731689439503.be59cfdc61180d35f564c8db0b26b902. after waiting 0 ms 2024-11-15T16:50:39,581 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731689439503.be59cfdc61180d35f564c8db0b26b902. 2024-11-15T16:50:39,582 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731689439503.be59cfdc61180d35f564c8db0b26b902. 2024-11-15T16:50:39,582 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for be59cfdc61180d35f564c8db0b26b902: Waiting for close lock at 1731689439581Disabling compacts and flushes for region at 1731689439581Disabling writes for close at 1731689439581Writing region close event to WAL at 1731689439582 (+1 ms)Closed at 1731689439582 2024-11-15T16:50:39,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741837_1013 (size=95) 2024-11-15T16:50:39,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741837_1013 (size=95) 2024-11-15T16:50:39,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741837_1013 (size=95) 2024-11-15T16:50:39,597 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-15T16:50:39,598 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-18595244:(num 1731689439513) 2024-11-15T16:50:39,601 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-15T16:50:39,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741839_1015 (size=320) 2024-11-15T16:50:39,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741839_1015 (size=320) 2024-11-15T16:50:39,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741839_1015 (size=320) 2024-11-15T16:50:39,626 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-15T16:50:39,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741840_1016 (size=253) 2024-11-15T16:50:39,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741840_1016 (size=253) 2024-11-15T16:50:39,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741840_1016 (size=253) 2024-11-15T16:50:39,663 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-1, size=320 (320bytes) 2024-11-15T16:50:39,664 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-15T16:50:39,664 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-15T16:50:39,664 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-1 2024-11-15T16:50:39,670 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-1 after 4ms 2024-11-15T16:50:39,677 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:39,678 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-1 took 15ms 2024-11-15T16:50:39,689 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-1 so closing down 2024-11-15T16:50:39,689 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T16:50:39,692 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-15T16:50:39,694 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000001-wal-1.temp 2024-11-15T16:50:39,695 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T16:50:39,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741841_1017 (size=320) 2024-11-15T16:50:39,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741841_1017 (size=320) 2024-11-15T16:50:39,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741841_1017 (size=320) 2024-11-15T16:50:39,710 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-15T16:50:39,713 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000002 2024-11-15T16:50:39,716 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-15T16:50:39,716 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-1, journal: Splitting hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-1, size=320 (320bytes) at 1731689439663Finishing writing output for hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-1 so closing down at 1731689439689 (+26 ms)Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000001-wal-1.temp at 1731689439694 (+5 ms)3 split writer threads finished at 1731689439695 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731689439710 (+15 ms)Rename recovered edits hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000002 at 1731689439713 (+3 ms)Processed 2 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1731689439716 (+3 ms) 2024-11-15T16:50:39,731 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-2, size=253 (253bytes) 2024-11-15T16:50:39,731 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-2 2024-11-15T16:50:39,732 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-2 after 0ms 2024-11-15T16:50:39,736 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:39,737 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-2 took 6ms 2024-11-15T16:50:39,739 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-2 so closing down 2024-11-15T16:50:39,739 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T16:50:39,742 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-15T16:50:39,744 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000002-wal-2.temp 2024-11-15T16:50:39,744 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T16:50:39,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741842_1018 (size=253) 2024-11-15T16:50:39,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741842_1018 (size=253) 2024-11-15T16:50:39,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741842_1018 (size=253) 2024-11-15T16:50:39,754 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-15T16:50:39,758 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:39,760 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000002-wal-2.temp, length=253 2024-11-15T16:50:39,762 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-15T16:50:39,762 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-2, journal: Splitting hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-2, size=253 (253bytes) at 1731689439731Finishing writing output for hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-2 so closing down at 1731689439739 (+8 ms)Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000002-wal-2.temp at 1731689439744 (+5 ms)3 split writer threads finished at 1731689439744Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731689439754 (+10 ms)Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1731689439762 (+8 ms) 2024-11-15T16:50:39,762 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:50:39,765 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:39,779 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal.1731689439766, exclude list is [], retry=0 2024-11-15T16:50:39,783 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:39,784 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:39,784 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:39,786 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal.1731689439766 2024-11-15T16:50:39,787 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:40741:40741)] 2024-11-15T16:50:39,787 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => be59cfdc61180d35f564c8db0b26b902, NAME => 'testReplayEditsWrittenIntoWAL,,1731689439503.be59cfdc61180d35f564c8db0b26b902.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:39,787 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731689439503.be59cfdc61180d35f564c8db0b26b902.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:39,787 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for be59cfdc61180d35f564c8db0b26b902 2024-11-15T16:50:39,787 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for be59cfdc61180d35f564c8db0b26b902 2024-11-15T16:50:39,791 INFO [StoreOpener-be59cfdc61180d35f564c8db0b26b902-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region be59cfdc61180d35f564c8db0b26b902 2024-11-15T16:50:39,792 INFO [StoreOpener-be59cfdc61180d35f564c8db0b26b902-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be59cfdc61180d35f564c8db0b26b902 columnFamilyName a 2024-11-15T16:50:39,792 DEBUG [StoreOpener-be59cfdc61180d35f564c8db0b26b902-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:39,794 INFO [StoreOpener-be59cfdc61180d35f564c8db0b26b902-1 {}] regionserver.HStore(327): Store=be59cfdc61180d35f564c8db0b26b902/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:39,794 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for be59cfdc61180d35f564c8db0b26b902 2024-11-15T16:50:39,795 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902 2024-11-15T16:50:39,798 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902 2024-11-15T16:50:39,799 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000002 2024-11-15T16:50:39,802 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:39,809 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000002 2024-11-15T16:50:39,812 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing be59cfdc61180d35f564c8db0b26b902 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-15T16:50:39,864 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/.tmp/a/9ff26d85b27548afb0c4fe186cec9349 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731689439598/Put/seqid=0 2024-11-15T16:50:39,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741844_1020 (size=5170) 2024-11-15T16:50:39,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741844_1020 (size=5170) 2024-11-15T16:50:39,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741844_1020 (size=5170) 2024-11-15T16:50:39,879 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/.tmp/a/9ff26d85b27548afb0c4fe186cec9349 2024-11-15T16:50:39,922 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/.tmp/a/9ff26d85b27548afb0c4fe186cec9349 as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/a/9ff26d85b27548afb0c4fe186cec9349 2024-11-15T16:50:39,933 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/a/9ff26d85b27548afb0c4fe186cec9349, entries=2, sequenceid=2, filesize=5.0 K 2024-11-15T16:50:39,940 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for be59cfdc61180d35f564c8db0b26b902 in 124ms, sequenceid=2, compaction requested=false; wal=null 2024-11-15T16:50:39,942 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/0000000000000000002 2024-11-15T16:50:39,943 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for be59cfdc61180d35f564c8db0b26b902 2024-11-15T16:50:39,943 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for be59cfdc61180d35f564c8db0b26b902 2024-11-15T16:50:39,947 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for be59cfdc61180d35f564c8db0b26b902 2024-11-15T16:50:39,950 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/be59cfdc61180d35f564c8db0b26b902/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-15T16:50:39,951 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened be59cfdc61180d35f564c8db0b26b902; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71689598, jitterRate=0.06825825572013855}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T16:50:39,953 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for be59cfdc61180d35f564c8db0b26b902: Writing region info on filesystem at 1731689439788Initializing all the Stores at 1731689439789 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689439789Obtaining lock to block concurrent updates at 1731689439812 (+23 ms)Preparing flush snapshotting stores in be59cfdc61180d35f564c8db0b26b902 at 1731689439812Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731689439503.be59cfdc61180d35f564c8db0b26b902., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731689439815 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731689439503.be59cfdc61180d35f564c8db0b26b902. at 1731689439815Flushing be59cfdc61180d35f564c8db0b26b902/a: creating writer at 1731689439817 (+2 ms)Flushing be59cfdc61180d35f564c8db0b26b902/a: appending metadata at 1731689439854 (+37 ms)Flushing be59cfdc61180d35f564c8db0b26b902/a: closing flushed file at 1731689439857 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69972cb2: reopening flushed file at 1731689439921 (+64 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for be59cfdc61180d35f564c8db0b26b902 in 124ms, sequenceid=2, compaction requested=false; wal=null at 1731689439940 (+19 ms)Cleaning up temporary data from old regions at 1731689439943 (+3 ms)Region opened successfully at 1731689439952 (+9 ms) 2024-11-15T16:50:39,985 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=378 (was 367) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:48458 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:60356 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:53482 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1263208986-172.17.0.2-1731689433335:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:42032 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:40387/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:43038 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:46588 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1263208986-172.17.0.2-1731689433335:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1263208986-172.17.0.2-1731689433335:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 260), ProcessCount=11 (was 11), AvailableMemoryMB=5632 (was 5651) 2024-11-15T16:50:39,997 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=378, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=260, ProcessCount=11, AvailableMemoryMB=5631 2024-11-15T16:50:40,014 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:40,017 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:40,018 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:50:40,022 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-62582790, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/hregion-62582790, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:40,036 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-62582790/hregion-62582790.1731689440023, exclude list is [], retry=0 2024-11-15T16:50:40,041 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:40,041 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:40,041 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:40,047 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-62582790/hregion-62582790.1731689440023 2024-11-15T16:50:40,047 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:36277:36277)] 2024-11-15T16:50:40,047 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 188c292c6194f4d875ea4061452d55e5, NAME => 'testReplayEditsWrittenIntoWAL,,1731689440015.188c292c6194f4d875ea4061452d55e5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40387/hbase 2024-11-15T16:50:40,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741846_1022 (size=64) 2024-11-15T16:50:40,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741846_1022 (size=64) 2024-11-15T16:50:40,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741846_1022 (size=64) 2024-11-15T16:50:40,060 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731689440015.188c292c6194f4d875ea4061452d55e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:40,062 INFO [StoreOpener-188c292c6194f4d875ea4061452d55e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 188c292c6194f4d875ea4061452d55e5 2024-11-15T16:50:40,065 INFO [StoreOpener-188c292c6194f4d875ea4061452d55e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 188c292c6194f4d875ea4061452d55e5 columnFamilyName a 2024-11-15T16:50:40,065 DEBUG [StoreOpener-188c292c6194f4d875ea4061452d55e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:40,066 INFO [StoreOpener-188c292c6194f4d875ea4061452d55e5-1 {}] regionserver.HStore(327): Store=188c292c6194f4d875ea4061452d55e5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:40,066 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 188c292c6194f4d875ea4061452d55e5 2024-11-15T16:50:40,067 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5 2024-11-15T16:50:40,068 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5 2024-11-15T16:50:40,069 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 188c292c6194f4d875ea4061452d55e5 2024-11-15T16:50:40,069 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 188c292c6194f4d875ea4061452d55e5 2024-11-15T16:50:40,071 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 188c292c6194f4d875ea4061452d55e5 2024-11-15T16:50:40,075 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:50:40,076 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 188c292c6194f4d875ea4061452d55e5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63341126, jitterRate=-0.05614367127418518}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T16:50:40,076 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 188c292c6194f4d875ea4061452d55e5: Writing region info on filesystem at 1731689440061Initializing all the Stores at 1731689440062 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689440062Cleaning up temporary data from old regions at 1731689440069 (+7 ms)Region opened successfully at 1731689440076 (+7 ms) 2024-11-15T16:50:40,076 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 188c292c6194f4d875ea4061452d55e5, disabling compactions & flushes 2024-11-15T16:50:40,076 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731689440015.188c292c6194f4d875ea4061452d55e5. 2024-11-15T16:50:40,076 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731689440015.188c292c6194f4d875ea4061452d55e5. 2024-11-15T16:50:40,076 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731689440015.188c292c6194f4d875ea4061452d55e5. after waiting 0 ms 2024-11-15T16:50:40,076 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731689440015.188c292c6194f4d875ea4061452d55e5. 2024-11-15T16:50:40,077 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731689440015.188c292c6194f4d875ea4061452d55e5. 2024-11-15T16:50:40,077 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 188c292c6194f4d875ea4061452d55e5: Waiting for close lock at 1731689440076Disabling compacts and flushes for region at 1731689440076Disabling writes for close at 1731689440076Writing region close event to WAL at 1731689440077 (+1 ms)Closed at 1731689440077 2024-11-15T16:50:40,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741845_1021 (size=95) 2024-11-15T16:50:40,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741845_1021 (size=95) 2024-11-15T16:50:40,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741845_1021 (size=95) 2024-11-15T16:50:40,084 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-15T16:50:40,085 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-62582790:(num 1731689440023) 2024-11-15T16:50:40,086 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-15T16:50:40,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741847_1023 (size=320) 2024-11-15T16:50:40,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741847_1023 (size=320) 2024-11-15T16:50:40,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741847_1023 (size=320) 2024-11-15T16:50:40,101 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-15T16:50:40,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741848_1024 (size=253) 2024-11-15T16:50:40,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741848_1024 (size=253) 2024-11-15T16:50:40,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741848_1024 (size=253) 2024-11-15T16:50:40,131 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-2, size=253 (253bytes) 2024-11-15T16:50:40,132 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-2 2024-11-15T16:50:40,133 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-2 after 0ms 2024-11-15T16:50:40,136 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:40,136 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-2 took 5ms 2024-11-15T16:50:40,139 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-2 so closing down 2024-11-15T16:50:40,139 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T16:50:40,141 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-15T16:50:40,143 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000002-wal-2.temp 2024-11-15T16:50:40,143 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T16:50:40,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741849_1025 (size=253) 2024-11-15T16:50:40,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741849_1025 (size=253) 2024-11-15T16:50:40,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741849_1025 (size=253) 2024-11-15T16:50:40,152 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-15T16:50:40,154 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000002 2024-11-15T16:50:40,154 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-15T16:50:40,154 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-2, journal: Splitting hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-2, size=253 (253bytes) at 1731689440132Finishing writing output for hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-2 so closing down at 1731689440139 (+7 ms)Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000002-wal-2.temp at 1731689440143 (+4 ms)3 split writer threads finished at 1731689440143Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731689440152 (+9 ms)Rename recovered edits hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000002 at 1731689440154 (+2 ms)Processed 1 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1731689440154 2024-11-15T16:50:40,169 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-1, size=320 (320bytes) 2024-11-15T16:50:40,169 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-1 2024-11-15T16:50:40,170 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-1 after 1ms 2024-11-15T16:50:40,173 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:40,174 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-1 took 5ms 2024-11-15T16:50:40,176 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-1 so closing down 2024-11-15T16:50:40,176 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T16:50:40,178 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-15T16:50:40,179 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000001-wal-1.temp 2024-11-15T16:50:40,180 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T16:50:40,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741850_1026 (size=320) 2024-11-15T16:50:40,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741850_1026 (size=320) 2024-11-15T16:50:40,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741850_1026 (size=320) 2024-11-15T16:50:40,188 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-15T16:50:40,192 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:40,195 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000002, length=253 2024-11-15T16:50:40,197 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000002 2024-11-15T16:50:40,197 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-15T16:50:40,197 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-1, journal: Splitting hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-1, size=320 (320bytes) at 1731689440169Finishing writing output for hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-1 so closing down at 1731689440176 (+7 ms)Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000001-wal-1.temp at 1731689440179 (+3 ms)3 split writer threads finished at 1731689440180 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731689440188 (+8 ms)Rename recovered edits hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000002 at 1731689440197 (+9 ms)Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1731689440197 2024-11-15T16:50:40,198 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:50:40,199 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:40,214 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal.1731689440200, exclude list is [], retry=0 2024-11-15T16:50:40,218 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:40,219 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:40,219 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:40,221 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal.1731689440200 2024-11-15T16:50:40,222 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:37569:37569)] 2024-11-15T16:50:40,222 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 188c292c6194f4d875ea4061452d55e5, NAME => 'testReplayEditsWrittenIntoWAL,,1731689440015.188c292c6194f4d875ea4061452d55e5.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:40,222 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731689440015.188c292c6194f4d875ea4061452d55e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:40,222 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 188c292c6194f4d875ea4061452d55e5 2024-11-15T16:50:40,222 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 188c292c6194f4d875ea4061452d55e5 2024-11-15T16:50:40,224 INFO [StoreOpener-188c292c6194f4d875ea4061452d55e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 188c292c6194f4d875ea4061452d55e5 2024-11-15T16:50:40,227 INFO [StoreOpener-188c292c6194f4d875ea4061452d55e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 188c292c6194f4d875ea4061452d55e5 columnFamilyName a 2024-11-15T16:50:40,227 DEBUG [StoreOpener-188c292c6194f4d875ea4061452d55e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:40,228 INFO [StoreOpener-188c292c6194f4d875ea4061452d55e5-1 {}] regionserver.HStore(327): Store=188c292c6194f4d875ea4061452d55e5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:40,228 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 188c292c6194f4d875ea4061452d55e5 2024-11-15T16:50:40,229 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5 2024-11-15T16:50:40,231 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5 2024-11-15T16:50:40,232 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000002 2024-11-15T16:50:40,236 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:40,237 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000002 2024-11-15T16:50:40,237 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 188c292c6194f4d875ea4061452d55e5 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-15T16:50:40,255 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/.tmp/a/454b63adde1a4586915b5623443ba512 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731689440085/Put/seqid=0 2024-11-15T16:50:40,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741852_1028 (size=5170) 2024-11-15T16:50:40,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741852_1028 (size=5170) 2024-11-15T16:50:40,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741852_1028 (size=5170) 2024-11-15T16:50:40,264 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/.tmp/a/454b63adde1a4586915b5623443ba512 2024-11-15T16:50:40,272 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/.tmp/a/454b63adde1a4586915b5623443ba512 as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/a/454b63adde1a4586915b5623443ba512 2024-11-15T16:50:40,279 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/a/454b63adde1a4586915b5623443ba512, entries=2, sequenceid=2, filesize=5.0 K 2024-11-15T16:50:40,279 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 188c292c6194f4d875ea4061452d55e5 in 42ms, sequenceid=2, compaction requested=false; wal=null 2024-11-15T16:50:40,280 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/0000000000000000002 2024-11-15T16:50:40,281 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 188c292c6194f4d875ea4061452d55e5 2024-11-15T16:50:40,281 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 188c292c6194f4d875ea4061452d55e5 2024-11-15T16:50:40,283 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 188c292c6194f4d875ea4061452d55e5 2024-11-15T16:50:40,286 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/188c292c6194f4d875ea4061452d55e5/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-15T16:50:40,287 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 188c292c6194f4d875ea4061452d55e5; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74547562, jitterRate=0.11084523797035217}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T16:50:40,287 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 188c292c6194f4d875ea4061452d55e5: Writing region info on filesystem at 1731689440222Initializing all the Stores at 1731689440224 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689440224Obtaining lock to block concurrent updates at 1731689440237 (+13 ms)Preparing flush snapshotting stores in 188c292c6194f4d875ea4061452d55e5 at 1731689440237Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731689440015.188c292c6194f4d875ea4061452d55e5., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731689440238 (+1 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731689440015.188c292c6194f4d875ea4061452d55e5. at 1731689440238Flushing 188c292c6194f4d875ea4061452d55e5/a: creating writer at 1731689440238Flushing 188c292c6194f4d875ea4061452d55e5/a: appending metadata at 1731689440254 (+16 ms)Flushing 188c292c6194f4d875ea4061452d55e5/a: closing flushed file at 1731689440254Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25164cf9: reopening flushed file at 1731689440271 (+17 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 188c292c6194f4d875ea4061452d55e5 in 42ms, sequenceid=2, compaction requested=false; wal=null at 1731689440279 (+8 ms)Cleaning up temporary data from old regions at 1731689440281 (+2 ms)Region opened successfully at 1731689440287 (+6 ms) 2024-11-15T16:50:40,304 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=388 (was 378) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:42092 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:48458 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1263208986-172.17.0.2-1731689433335:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:53572 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1263208986-172.17.0.2-1731689433335:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:60356 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:43038 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1263208986-172.17.0.2-1731689433335:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:46676 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=287 (was 260) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5628 (was 5631) 2024-11-15T16:50:40,316 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=388, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=287, ProcessCount=11, AvailableMemoryMB=5628 2024-11-15T16:50:40,331 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:40,334 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:40,335 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:50:40,338 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-69628422, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/hregion-69628422, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:40,351 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-69628422/hregion-69628422.1731689440338, exclude list is [], retry=0 2024-11-15T16:50:40,355 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:40,355 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:40,355 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:40,360 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-69628422/hregion-69628422.1731689440338 2024-11-15T16:50:40,361 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:36277:36277)] 2024-11-15T16:50:40,361 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a2763ad959188a8ba144eadea1bd6da5, NAME => 'testReplayEditsWrittenIntoWAL,,1731689440332.a2763ad959188a8ba144eadea1bd6da5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40387/hbase 2024-11-15T16:50:40,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741854_1030 (size=64) 2024-11-15T16:50:40,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741854_1030 (size=64) 2024-11-15T16:50:40,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741854_1030 (size=64) 2024-11-15T16:50:40,373 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731689440332.a2763ad959188a8ba144eadea1bd6da5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:40,375 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:40,377 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2763ad959188a8ba144eadea1bd6da5 columnFamilyName a 2024-11-15T16:50:40,377 DEBUG [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:40,378 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] regionserver.HStore(327): Store=a2763ad959188a8ba144eadea1bd6da5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:40,378 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:40,379 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2763ad959188a8ba144eadea1bd6da5 columnFamilyName b 2024-11-15T16:50:40,379 DEBUG [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:40,380 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] regionserver.HStore(327): Store=a2763ad959188a8ba144eadea1bd6da5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:40,380 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:40,381 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2763ad959188a8ba144eadea1bd6da5 columnFamilyName c 2024-11-15T16:50:40,381 DEBUG [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:40,382 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] regionserver.HStore(327): Store=a2763ad959188a8ba144eadea1bd6da5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:40,382 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:40,383 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:40,383 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:40,385 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:40,385 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:40,385 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:50:40,387 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:40,389 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:50:40,390 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a2763ad959188a8ba144eadea1bd6da5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72340199, jitterRate=0.07795296609401703}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:50:40,390 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a2763ad959188a8ba144eadea1bd6da5: Writing region info on filesystem at 1731689440374Initializing all the Stores at 1731689440375 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689440375Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689440375Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689440375Cleaning up temporary data from old regions at 1731689440385 (+10 ms)Region opened successfully at 1731689440390 (+5 ms) 2024-11-15T16:50:40,390 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a2763ad959188a8ba144eadea1bd6da5, disabling compactions & flushes 2024-11-15T16:50:40,390 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731689440332.a2763ad959188a8ba144eadea1bd6da5. 2024-11-15T16:50:40,391 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731689440332.a2763ad959188a8ba144eadea1bd6da5. 2024-11-15T16:50:40,391 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731689440332.a2763ad959188a8ba144eadea1bd6da5. after waiting 0 ms 2024-11-15T16:50:40,391 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731689440332.a2763ad959188a8ba144eadea1bd6da5. 2024-11-15T16:50:40,391 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731689440332.a2763ad959188a8ba144eadea1bd6da5. 2024-11-15T16:50:40,391 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a2763ad959188a8ba144eadea1bd6da5: Waiting for close lock at 1731689440390Disabling compacts and flushes for region at 1731689440390Disabling writes for close at 1731689440391 (+1 ms)Writing region close event to WAL at 1731689440391Closed at 1731689440391 2024-11-15T16:50:40,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741853_1029 (size=95) 2024-11-15T16:50:40,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741853_1029 (size=95) 2024-11-15T16:50:40,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741853_1029 (size=95) 2024-11-15T16:50:40,399 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-15T16:50:40,399 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-69628422:(num 1731689440338) 2024-11-15T16:50:40,399 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:50:40,401 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:40,416 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402, exclude list is [], retry=0 2024-11-15T16:50:40,419 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:40,420 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:40,420 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:40,423 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402 2024-11-15T16:50:40,423 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:36277:36277)] 2024-11-15T16:50:40,632 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402, size=0 (0bytes) 2024-11-15T16:50:40,632 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402 might be still open, length is 0 2024-11-15T16:50:40,632 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402 2024-11-15T16:50:40,633 WARN [IPC Server handler 0 on default port 40387 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-15T16:50:40,634 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402 after 2ms 2024-11-15T16:50:41,438 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:53600 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39325:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53600 dst: /127.0.0.1:39325 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39325 remote=/127.0.0.1:53600]. Total timeout mills is 60000, 59160 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:50:41,439 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:42130 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37733:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42130 dst: /127.0.0.1:37733 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:50:41,439 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:46700 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:42975:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46700 dst: /127.0.0.1:42975 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:50:41,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741855_1032 (size=263633) 2024-11-15T16:50:41,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741855_1032 (size=263633) 2024-11-15T16:50:41,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741855_1032 (size=263633) 2024-11-15T16:50:44,630 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T16:50:44,635 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402 after 4003ms 2024-11-15T16:50:44,639 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:44,641 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402 took 4009ms 2024-11-15T16:50:44,647 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731689440402.temp 2024-11-15T16:50:44,649 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/recovered.edits/0000000000000000001-wal.1731689440402.temp 2024-11-15T16:50:44,700 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T16:50:44,810 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402; continuing. 2024-11-15T16:50:44,810 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402 so closing down 2024-11-15T16:50:44,810 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T16:50:44,811 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T16:50:44,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741856_1033 (size=263641) 2024-11-15T16:50:44,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741856_1033 (size=263641) 2024-11-15T16:50:44,816 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/recovered.edits/0000000000000000001-wal.1731689440402.temp (wrote 3002 edits, skipped 0 edits in 80 ms) 2024-11-15T16:50:44,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741856_1033 (size=263641) 2024-11-15T16:50:44,818 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/recovered.edits/0000000000000000001-wal.1731689440402.temp to hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/recovered.edits/0000000000000003002 2024-11-15T16:50:44,818 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 177 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402, size=0, length=0, corrupted=false, cancelled=false 2024-11-15T16:50:44,819 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402, journal: Splitting hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402, size=0 (0bytes) at 1731689440632Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/recovered.edits/0000000000000000001-wal.1731689440402.temp at 1731689444649 (+4017 ms)Split 1024 edits, skipped 0 edits. at 1731689444736 (+87 ms)Split 2048 edits, skipped 0 edits. at 1731689444775 (+39 ms)Finishing writing output for hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402 so closing down at 1731689444810 (+35 ms)3 split writer threads finished at 1731689444811 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/recovered.edits/0000000000000000001-wal.1731689440402.temp (wrote 3002 edits, skipped 0 edits in 80 ms) at 1731689444816 (+5 ms)Rename recovered edits hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/recovered.edits/0000000000000000001-wal.1731689440402.temp to hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/recovered.edits/0000000000000003002 at 1731689444818 (+2 ms)Processed 3002 edits across 1 Regions in 177 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402, size=0, length=0, corrupted=false, cancelled=false at 1731689444818 2024-11-15T16:50:44,821 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402 to hdfs://localhost:40387/hbase/oldWALs/wal.1731689440402 2024-11-15T16:50:44,823 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/recovered.edits/0000000000000003002 2024-11-15T16:50:44,823 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:50:44,826 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:44,848 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689444827, exclude list is [], retry=0 2024-11-15T16:50:44,851 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:44,852 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:44,852 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:44,854 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689444827 2024-11-15T16:50:44,855 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:40741:40741)] 2024-11-15T16:50:44,855 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731689440332.a2763ad959188a8ba144eadea1bd6da5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:44,857 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:44,858 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2763ad959188a8ba144eadea1bd6da5 columnFamilyName a 2024-11-15T16:50:44,858 DEBUG [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:44,858 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] regionserver.HStore(327): Store=a2763ad959188a8ba144eadea1bd6da5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:44,859 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:44,860 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2763ad959188a8ba144eadea1bd6da5 columnFamilyName b 2024-11-15T16:50:44,860 DEBUG [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:44,860 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] regionserver.HStore(327): Store=a2763ad959188a8ba144eadea1bd6da5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:44,861 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:44,862 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2763ad959188a8ba144eadea1bd6da5 columnFamilyName c 2024-11-15T16:50:44,862 DEBUG [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:44,862 INFO [StoreOpener-a2763ad959188a8ba144eadea1bd6da5-1 {}] regionserver.HStore(327): Store=a2763ad959188a8ba144eadea1bd6da5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:44,863 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:44,863 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:44,865 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:44,866 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/recovered.edits/0000000000000003002 2024-11-15T16:50:44,869 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:44,905 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-15T16:50:45,275 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a2763ad959188a8ba144eadea1bd6da5 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-15T16:50:45,312 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/a/8c8b6fded5fd4f958bc52569f7da2b73 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731689440434/Put/seqid=0 2024-11-15T16:50:45,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741858_1035 (size=50463) 2024-11-15T16:50:45,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741858_1035 (size=50463) 2024-11-15T16:50:45,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741858_1035 (size=50463) 2024-11-15T16:50:45,323 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/a/8c8b6fded5fd4f958bc52569f7da2b73 2024-11-15T16:50:45,333 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/a/8c8b6fded5fd4f958bc52569f7da2b73 as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/a/8c8b6fded5fd4f958bc52569f7da2b73 2024-11-15T16:50:45,340 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/a/8c8b6fded5fd4f958bc52569f7da2b73, entries=754, sequenceid=754, filesize=49.3 K 2024-11-15T16:50:45,340 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for a2763ad959188a8ba144eadea1bd6da5 in 65ms, sequenceid=754, compaction requested=false; wal=null 2024-11-15T16:50:45,364 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-15T16:50:45,364 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a2763ad959188a8ba144eadea1bd6da5 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-15T16:50:45,373 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/a/fe48c75f9c414875bc1266d57a2d54d6 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731689440496/Put/seqid=0 2024-11-15T16:50:45,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741859_1036 (size=20072) 2024-11-15T16:50:45,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741859_1036 (size=20072) 2024-11-15T16:50:45,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741859_1036 (size=20072) 2024-11-15T16:50:45,383 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/a/fe48c75f9c414875bc1266d57a2d54d6 2024-11-15T16:50:45,412 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/b/1184867951564a15bd4794af80a3be41 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731689440522/Put/seqid=0 2024-11-15T16:50:45,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741860_1037 (size=35835) 2024-11-15T16:50:45,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741860_1037 (size=35835) 2024-11-15T16:50:45,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741860_1037 (size=35835) 2024-11-15T16:50:45,422 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/b/1184867951564a15bd4794af80a3be41 2024-11-15T16:50:45,429 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/a/fe48c75f9c414875bc1266d57a2d54d6 as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/a/fe48c75f9c414875bc1266d57a2d54d6 2024-11-15T16:50:45,435 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/a/fe48c75f9c414875bc1266d57a2d54d6, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-15T16:50:45,437 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/b/1184867951564a15bd4794af80a3be41 as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/b/1184867951564a15bd4794af80a3be41 2024-11-15T16:50:45,444 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/b/1184867951564a15bd4794af80a3be41, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-15T16:50:45,444 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for a2763ad959188a8ba144eadea1bd6da5 in 80ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-15T16:50:45,456 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-15T16:50:45,457 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a2763ad959188a8ba144eadea1bd6da5 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-15T16:50:45,464 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/b/f079a206eef446d48090aafa060c6e07 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731689440536/Put/seqid=0 2024-11-15T16:50:45,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741861_1038 (size=35082) 2024-11-15T16:50:45,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741861_1038 (size=35082) 2024-11-15T16:50:45,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741861_1038 (size=35082) 2024-11-15T16:50:45,476 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/b/f079a206eef446d48090aafa060c6e07 2024-11-15T16:50:45,500 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/c/14a46164023049b58e696fcda3d509e7 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731689440559/Put/seqid=0 2024-11-15T16:50:45,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741862_1039 (size=20825) 2024-11-15T16:50:45,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741862_1039 (size=20825) 2024-11-15T16:50:45,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741862_1039 (size=20825) 2024-11-15T16:50:45,509 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/c/14a46164023049b58e696fcda3d509e7 2024-11-15T16:50:45,516 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/b/f079a206eef446d48090aafa060c6e07 as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/b/f079a206eef446d48090aafa060c6e07 2024-11-15T16:50:45,523 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/b/f079a206eef446d48090aafa060c6e07, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-15T16:50:45,524 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/c/14a46164023049b58e696fcda3d509e7 as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/c/14a46164023049b58e696fcda3d509e7 2024-11-15T16:50:45,531 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/c/14a46164023049b58e696fcda3d509e7, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-15T16:50:45,532 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for a2763ad959188a8ba144eadea1bd6da5 in 74ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-15T16:50:45,547 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731689440596/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731689440332.a2763ad959188a8ba144eadea1bd6da5. 2024-11-15T16:50:45,551 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/recovered.edits/0000000000000003002 2024-11-15T16:50:45,552 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-15T16:50:45,552 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a2763ad959188a8ba144eadea1bd6da5 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-15T16:50:45,563 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/c/34533e3f88ff4faa96932454fa5e06d9 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731689440566/Put/seqid=0 2024-11-15T16:50:45,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741863_1040 (size=50301) 2024-11-15T16:50:45,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741863_1040 (size=50301) 2024-11-15T16:50:45,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741863_1040 (size=50301) 2024-11-15T16:50:45,578 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/c/34533e3f88ff4faa96932454fa5e06d9 2024-11-15T16:50:45,586 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 34533e3f88ff4faa96932454fa5e06d9 2024-11-15T16:50:45,588 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/.tmp/c/34533e3f88ff4faa96932454fa5e06d9 as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/c/34533e3f88ff4faa96932454fa5e06d9 2024-11-15T16:50:45,596 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 34533e3f88ff4faa96932454fa5e06d9 2024-11-15T16:50:45,596 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/c/34533e3f88ff4faa96932454fa5e06d9, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-15T16:50:45,597 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for a2763ad959188a8ba144eadea1bd6da5 in 45ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-15T16:50:45,598 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/recovered.edits/0000000000000003002 2024-11-15T16:50:45,600 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:45,600 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:45,601 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T16:50:45,603 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a2763ad959188a8ba144eadea1bd6da5 2024-11-15T16:50:45,607 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenIntoWAL/a2763ad959188a8ba144eadea1bd6da5/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-15T16:50:45,608 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a2763ad959188a8ba144eadea1bd6da5; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60266113, jitterRate=-0.10196493566036224}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T16:50:45,608 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a2763ad959188a8ba144eadea1bd6da5: Writing region info on filesystem at 1731689444855Initializing all the Stores at 1731689444856 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689444856Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689444856Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689444856Cleaning up temporary data from old regions at 1731689445600 (+744 ms)Region opened successfully at 1731689445608 (+8 ms) 2024-11-15T16:50:45,663 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a2763ad959188a8ba144eadea1bd6da5, disabling compactions & flushes 2024-11-15T16:50:45,663 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731689440332.a2763ad959188a8ba144eadea1bd6da5. 2024-11-15T16:50:45,663 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731689440332.a2763ad959188a8ba144eadea1bd6da5. 2024-11-15T16:50:45,663 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731689440332.a2763ad959188a8ba144eadea1bd6da5. after waiting 0 ms 2024-11-15T16:50:45,663 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731689440332.a2763ad959188a8ba144eadea1bd6da5. 2024-11-15T16:50:45,666 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731689440332.a2763ad959188a8ba144eadea1bd6da5. 2024-11-15T16:50:45,666 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a2763ad959188a8ba144eadea1bd6da5: Waiting for close lock at 1731689445663Disabling compacts and flushes for region at 1731689445663Disabling writes for close at 1731689445663Writing region close event to WAL at 1731689445666 (+3 ms)Closed at 1731689445666 2024-11-15T16:50:45,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741857_1034 (size=95) 2024-11-15T16:50:45,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741857_1034 (size=95) 2024-11-15T16:50:45,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741857_1034 (size=95) 2024-11-15T16:50:45,673 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-15T16:50:45,673 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731689444827) 2024-11-15T16:50:45,692 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=404 (was 388) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1036529220_22 at /127.0.0.1:46750 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:40387 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1036529220_22 at /127.0.0.1:46730 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1396088596) connection to localhost/127.0.0.1:41293 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1036529220_22 at /127.0.0.1:53616 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41293 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1036529220_22 at /127.0.0.1:42180 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42261 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1396088596) connection to localhost/127.0.0.1:42261 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1396088596) connection to localhost/127.0.0.1:40387 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=272 (was 287), ProcessCount=11 (was 11), AvailableMemoryMB=5420 (was 5628) 2024-11-15T16:50:45,704 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=404, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=272, ProcessCount=11, AvailableMemoryMB=5419 2024-11-15T16:50:45,720 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:45,723 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:45,724 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:50:45,727 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-81327583, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/hregion-81327583, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:45,741 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-81327583/hregion-81327583.1731689445728, exclude list is [], retry=0 2024-11-15T16:50:45,744 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:45,745 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:45,745 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:45,748 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-81327583/hregion-81327583.1731689445728 2024-11-15T16:50:45,748 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:40741:40741)] 2024-11-15T16:50:45,748 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c93e5ab21efe86e5f39fd10efe94db96, NAME => 'test2727,,1731689445721.c93e5ab21efe86e5f39fd10efe94db96.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40387/hbase 2024-11-15T16:50:45,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741865_1042 (size=43) 2024-11-15T16:50:45,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741865_1042 (size=43) 2024-11-15T16:50:45,760 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731689445721.c93e5ab21efe86e5f39fd10efe94db96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:45,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741865_1042 (size=43) 2024-11-15T16:50:45,762 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:45,763 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c93e5ab21efe86e5f39fd10efe94db96 columnFamilyName a 2024-11-15T16:50:45,763 DEBUG [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:45,764 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] regionserver.HStore(327): Store=c93e5ab21efe86e5f39fd10efe94db96/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:45,764 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:45,765 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c93e5ab21efe86e5f39fd10efe94db96 columnFamilyName b 2024-11-15T16:50:45,765 DEBUG [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:45,766 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] regionserver.HStore(327): Store=c93e5ab21efe86e5f39fd10efe94db96/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:45,766 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:45,767 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c93e5ab21efe86e5f39fd10efe94db96 columnFamilyName c 2024-11-15T16:50:45,767 DEBUG [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:45,768 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] regionserver.HStore(327): Store=c93e5ab21efe86e5f39fd10efe94db96/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:45,768 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:45,769 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:45,769 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:45,770 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:45,770 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:45,771 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:50:45,773 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:45,775 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:50:45,776 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c93e5ab21efe86e5f39fd10efe94db96; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59659919, jitterRate=-0.11099793016910553}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:50:45,777 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c93e5ab21efe86e5f39fd10efe94db96: Writing region info on filesystem at 1731689445760Initializing all the Stores at 1731689445761 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689445761Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689445761Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689445761Cleaning up temporary data from old regions at 1731689445770 (+9 ms)Region opened successfully at 1731689445777 (+7 ms) 2024-11-15T16:50:45,777 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c93e5ab21efe86e5f39fd10efe94db96, disabling compactions & flushes 2024-11-15T16:50:45,777 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731689445721.c93e5ab21efe86e5f39fd10efe94db96. 2024-11-15T16:50:45,777 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731689445721.c93e5ab21efe86e5f39fd10efe94db96. 2024-11-15T16:50:45,777 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731689445721.c93e5ab21efe86e5f39fd10efe94db96. after waiting 0 ms 2024-11-15T16:50:45,777 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731689445721.c93e5ab21efe86e5f39fd10efe94db96. 2024-11-15T16:50:45,777 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731689445721.c93e5ab21efe86e5f39fd10efe94db96. 2024-11-15T16:50:45,778 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c93e5ab21efe86e5f39fd10efe94db96: Waiting for close lock at 1731689445777Disabling compacts and flushes for region at 1731689445777Disabling writes for close at 1731689445777Writing region close event to WAL at 1731689445777Closed at 1731689445777 2024-11-15T16:50:45,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741864_1041 (size=95) 2024-11-15T16:50:45,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741864_1041 (size=95) 2024-11-15T16:50:45,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741864_1041 (size=95) 2024-11-15T16:50:45,786 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-15T16:50:45,786 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-81327583:(num 1731689445728) 2024-11-15T16:50:45,786 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:50:45,788 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:45,805 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689445789, exclude list is [], retry=0 2024-11-15T16:50:45,808 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:45,808 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:45,809 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:45,811 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689445789 2024-11-15T16:50:45,811 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:40741:40741)] 2024-11-15T16:50:45,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741866_1043 (size=263359) 2024-11-15T16:50:45,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741866_1043 (size=263359) 2024-11-15T16:50:45,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741866_1043 (size=263359) 2024-11-15T16:50:45,999 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689445789, size=257.2 K (263359bytes) 2024-11-15T16:50:45,999 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689445789 2024-11-15T16:50:45,999 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689445789 after 0ms 2024-11-15T16:50:46,003 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689445789: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:46,005 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689445789 took 7ms 2024-11-15T16:50:46,010 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731689445789.temp 2024-11-15T16:50:46,012 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000000001-wal.1731689445789.temp 2024-11-15T16:50:46,082 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689445789 so closing down 2024-11-15T16:50:46,082 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T16:50:46,082 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T16:50:46,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741867_1044 (size=263359) 2024-11-15T16:50:46,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741867_1044 (size=263359) 2024-11-15T16:50:46,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741867_1044 (size=263359) 2024-11-15T16:50:46,087 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000000001-wal.1731689445789.temp (wrote 3000 edits, skipped 0 edits in 51 ms) 2024-11-15T16:50:46,089 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000000001-wal.1731689445789.temp to hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000003000 2024-11-15T16:50:46,089 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 84 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689445789, size=257.2 K, length=263359, corrupted=false, cancelled=false 2024-11-15T16:50:46,089 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689445789, journal: Splitting hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689445789, size=257.2 K (263359bytes) at 1731689445999Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000000001-wal.1731689445789.temp at 1731689446012 (+13 ms)Split 1024 edits, skipped 0 edits. at 1731689446033 (+21 ms)Split 2048 edits, skipped 0 edits. at 1731689446059 (+26 ms)Finishing writing output for hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689445789 so closing down at 1731689446082 (+23 ms)3 split writer threads finished at 1731689446082Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000000001-wal.1731689445789.temp (wrote 3000 edits, skipped 0 edits in 51 ms) at 1731689446087 (+5 ms)Rename recovered edits hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000000001-wal.1731689445789.temp to hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000003000 at 1731689446089 (+2 ms)Processed 3000 edits across 1 Regions in 84 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689445789, size=257.2 K, length=263359, corrupted=false, cancelled=false at 1731689446089 2024-11-15T16:50:46,092 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689445789 to hdfs://localhost:40387/hbase/oldWALs/wal.1731689445789 2024-11-15T16:50:46,093 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000003000 2024-11-15T16:50:46,093 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:50:46,096 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:46,118 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689446097, exclude list is [], retry=0 2024-11-15T16:50:46,122 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:46,123 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:46,123 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:46,125 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689446097 2024-11-15T16:50:46,126 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:40741:40741)] 2024-11-15T16:50:46,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741868_1045 (size=263486) 2024-11-15T16:50:46,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741868_1045 (size=263486) 2024-11-15T16:50:46,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741868_1045 (size=263486) 2024-11-15T16:50:46,347 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689446097, size=257.3 K (263486bytes) 2024-11-15T16:50:46,347 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689446097 2024-11-15T16:50:46,348 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689446097 after 1ms 2024-11-15T16:50:46,351 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689446097: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:46,353 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689446097 took 6ms 2024-11-15T16:50:46,358 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731689446097.temp 2024-11-15T16:50:46,360 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000003001-wal.1731689446097.temp 2024-11-15T16:50:46,417 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689446097 so closing down 2024-11-15T16:50:46,418 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T16:50:46,418 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T16:50:46,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741869_1046 (size=263486) 2024-11-15T16:50:46,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741869_1046 (size=263486) 2024-11-15T16:50:46,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741869_1046 (size=263486) 2024-11-15T16:50:46,422 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000003001-wal.1731689446097.temp (wrote 3000 edits, skipped 0 edits in 44 ms) 2024-11-15T16:50:46,424 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000003001-wal.1731689446097.temp to hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000006000 2024-11-15T16:50:46,424 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689446097, size=257.3 K, length=263486, corrupted=false, cancelled=false 2024-11-15T16:50:46,424 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689446097, journal: Splitting hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689446097, size=257.3 K (263486bytes) at 1731689446347Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000003001-wal.1731689446097.temp at 1731689446360 (+13 ms)Split 1024 edits, skipped 0 edits. at 1731689446376 (+16 ms)Split 2048 edits, skipped 0 edits. at 1731689446398 (+22 ms)Finishing writing output for hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689446097 so closing down at 1731689446418 (+20 ms)3 split writer threads finished at 1731689446418Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000003001-wal.1731689446097.temp (wrote 3000 edits, skipped 0 edits in 44 ms) at 1731689446422 (+4 ms)Rename recovered edits hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000003001-wal.1731689446097.temp to hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000006000 at 1731689446424 (+2 ms)Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689446097, size=257.3 K, length=263486, corrupted=false, cancelled=false at 1731689446424 2024-11-15T16:50:46,426 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689446097 to hdfs://localhost:40387/hbase/oldWALs/wal.1731689446097 2024-11-15T16:50:46,427 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000006000 2024-11-15T16:50:46,428 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:50:46,430 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/test2727-manual,16010,1731689445720, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:46,445 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689446430, exclude list is [], retry=0 2024-11-15T16:50:46,449 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:46,449 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:46,450 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:46,452 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731689445720/wal.1731689446430 2024-11-15T16:50:46,452 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:36277:36277)] 2024-11-15T16:50:46,452 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c93e5ab21efe86e5f39fd10efe94db96, NAME => 'test2727,,1731689445721.c93e5ab21efe86e5f39fd10efe94db96.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:46,453 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731689445721.c93e5ab21efe86e5f39fd10efe94db96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:46,453 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:46,453 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:46,455 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:46,456 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c93e5ab21efe86e5f39fd10efe94db96 columnFamilyName a 2024-11-15T16:50:46,456 DEBUG [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:46,456 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] regionserver.HStore(327): Store=c93e5ab21efe86e5f39fd10efe94db96/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:46,457 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:46,458 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c93e5ab21efe86e5f39fd10efe94db96 columnFamilyName b 2024-11-15T16:50:46,458 DEBUG [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:46,458 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] regionserver.HStore(327): Store=c93e5ab21efe86e5f39fd10efe94db96/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:46,459 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:46,460 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c93e5ab21efe86e5f39fd10efe94db96 columnFamilyName c 2024-11-15T16:50:46,460 DEBUG [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:46,460 INFO [StoreOpener-c93e5ab21efe86e5f39fd10efe94db96-1 {}] regionserver.HStore(327): Store=c93e5ab21efe86e5f39fd10efe94db96/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:46,461 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:46,462 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:46,464 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:46,465 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000003000 2024-11-15T16:50:46,468 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:46,547 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000003000 2024-11-15T16:50:46,549 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000006000 2024-11-15T16:50:46,552 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:46,606 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000006000 2024-11-15T16:50:46,606 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c93e5ab21efe86e5f39fd10efe94db96 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-15T16:50:46,630 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/.tmp/a/be67b1d67cf04201b9d81daa4aa8f995 is 41, key is test2727/a:100/1731689446133/Put/seqid=0 2024-11-15T16:50:46,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741871_1048 (size=84227) 2024-11-15T16:50:46,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741871_1048 (size=84227) 2024-11-15T16:50:46,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741871_1048 (size=84227) 2024-11-15T16:50:46,637 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/.tmp/a/be67b1d67cf04201b9d81daa4aa8f995 2024-11-15T16:50:46,673 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/.tmp/b/05c9cd2acc7740ef970b4b9e01f11eaf is 41, key is test2727/b:100/1731689446189/Put/seqid=0 2024-11-15T16:50:46,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741872_1049 (size=84609) 2024-11-15T16:50:46,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741872_1049 (size=84609) 2024-11-15T16:50:46,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741872_1049 (size=84609) 2024-11-15T16:50:46,680 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/.tmp/b/05c9cd2acc7740ef970b4b9e01f11eaf 2024-11-15T16:50:46,718 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/.tmp/c/fb9f3123bae94cc1a5cd3f73b4690266 is 41, key is test2727/c:100/1731689446245/Put/seqid=0 2024-11-15T16:50:46,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741873_1050 (size=84609) 2024-11-15T16:50:46,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741873_1050 (size=84609) 2024-11-15T16:50:46,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741873_1050 (size=84609) 2024-11-15T16:50:46,725 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/.tmp/c/fb9f3123bae94cc1a5cd3f73b4690266 2024-11-15T16:50:46,733 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/.tmp/a/be67b1d67cf04201b9d81daa4aa8f995 as hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/a/be67b1d67cf04201b9d81daa4aa8f995 2024-11-15T16:50:46,739 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/a/be67b1d67cf04201b9d81daa4aa8f995, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-15T16:50:46,741 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/.tmp/b/05c9cd2acc7740ef970b4b9e01f11eaf as hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/b/05c9cd2acc7740ef970b4b9e01f11eaf 2024-11-15T16:50:46,748 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/b/05c9cd2acc7740ef970b4b9e01f11eaf, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-15T16:50:46,749 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/.tmp/c/fb9f3123bae94cc1a5cd3f73b4690266 as hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/c/fb9f3123bae94cc1a5cd3f73b4690266 2024-11-15T16:50:46,785 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/c/fb9f3123bae94cc1a5cd3f73b4690266, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-15T16:50:46,786 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for c93e5ab21efe86e5f39fd10efe94db96 in 180ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-15T16:50:46,787 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000003000 2024-11-15T16:50:46,787 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/0000000000000006000 2024-11-15T16:50:46,789 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:46,789 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:46,789 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:50:46,791 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c93e5ab21efe86e5f39fd10efe94db96 2024-11-15T16:50:46,795 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/test2727/c93e5ab21efe86e5f39fd10efe94db96/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-15T16:50:46,796 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c93e5ab21efe86e5f39fd10efe94db96; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74609218, jitterRate=0.11176398396492004}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:50:46,797 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c93e5ab21efe86e5f39fd10efe94db96: Writing region info on filesystem at 1731689446453Initializing all the Stores at 1731689446454 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689446454Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689446454Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689446454Obtaining lock to block concurrent updates at 1731689446606 (+152 ms)Preparing flush snapshotting stores in c93e5ab21efe86e5f39fd10efe94db96 at 1731689446606Finished memstore snapshotting test2727,,1731689445721.c93e5ab21efe86e5f39fd10efe94db96., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731689446606Flushing stores of test2727,,1731689445721.c93e5ab21efe86e5f39fd10efe94db96. at 1731689446606Flushing c93e5ab21efe86e5f39fd10efe94db96/a: creating writer at 1731689446606Flushing c93e5ab21efe86e5f39fd10efe94db96/a: appending metadata at 1731689446629 (+23 ms)Flushing c93e5ab21efe86e5f39fd10efe94db96/a: closing flushed file at 1731689446629Flushing c93e5ab21efe86e5f39fd10efe94db96/b: creating writer at 1731689446645 (+16 ms)Flushing c93e5ab21efe86e5f39fd10efe94db96/b: appending metadata at 1731689446671 (+26 ms)Flushing c93e5ab21efe86e5f39fd10efe94db96/b: closing flushed file at 1731689446671Flushing c93e5ab21efe86e5f39fd10efe94db96/c: creating writer at 1731689446688 (+17 ms)Flushing c93e5ab21efe86e5f39fd10efe94db96/c: appending metadata at 1731689446717 (+29 ms)Flushing c93e5ab21efe86e5f39fd10efe94db96/c: closing flushed file at 1731689446717Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46d1b802: reopening flushed file at 1731689446732 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6812d65c: reopening flushed file at 1731689446740 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@364dfb5b: reopening flushed file at 1731689446748 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for c93e5ab21efe86e5f39fd10efe94db96 in 180ms, sequenceid=6000, compaction requested=false; wal=null at 1731689446786 (+38 ms)Cleaning up temporary data from old regions at 1731689446789 (+3 ms)Region opened successfully at 1731689446797 (+8 ms) 2024-11-15T16:50:46,799 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-15T16:50:46,799 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c93e5ab21efe86e5f39fd10efe94db96, disabling compactions & flushes 2024-11-15T16:50:46,799 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731689445721.c93e5ab21efe86e5f39fd10efe94db96. 2024-11-15T16:50:46,799 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731689445721.c93e5ab21efe86e5f39fd10efe94db96. 2024-11-15T16:50:46,799 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731689445721.c93e5ab21efe86e5f39fd10efe94db96. after waiting 0 ms 2024-11-15T16:50:46,799 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731689445721.c93e5ab21efe86e5f39fd10efe94db96. 2024-11-15T16:50:46,803 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731689445721.c93e5ab21efe86e5f39fd10efe94db96. 2024-11-15T16:50:46,803 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c93e5ab21efe86e5f39fd10efe94db96: Waiting for close lock at 1731689446799Disabling compacts and flushes for region at 1731689446799Disabling writes for close at 1731689446799Writing region close event to WAL at 1731689446803 (+4 ms)Closed at 1731689446803 2024-11-15T16:50:46,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741870_1047 (size=95) 2024-11-15T16:50:46,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741870_1047 (size=95) 2024-11-15T16:50:46,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741870_1047 (size=95) 2024-11-15T16:50:46,813 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-15T16:50:46,813 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731689446430) 2024-11-15T16:50:46,832 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=406 (was 404) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:46750 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:53760 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=272 (was 272), ProcessCount=11 (was 11), AvailableMemoryMB=5146 (was 5419) 2024-11-15T16:50:46,845 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=406, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=272, ProcessCount=11, AvailableMemoryMB=5145 2024-11-15T16:50:46,860 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:46,864 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T16:50:46,865 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-15T16:50:46,867 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T16:50:46,867 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-15T16:50:46,867 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731689446860, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:46,868 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T16:50:46,868 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-15T16:50:46,868 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731689446868 2024-11-15T16:50:46,868 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-15T16:50:46,868 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-15T16:50:46,878 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731689446860/wal.1731689446868 2024-11-15T16:50:46,882 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:40741:40741)] 2024-11-15T16:50:46,886 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5cebc9f67fc09e816de8050ffbc2bbfc, NAME => 'testSequentialEditLogSeqNum,,1731689446861.5cebc9f67fc09e816de8050ffbc2bbfc.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:46,886 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731689446861.5cebc9f67fc09e816de8050ffbc2bbfc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:46,886 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5cebc9f67fc09e816de8050ffbc2bbfc 2024-11-15T16:50:46,887 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5cebc9f67fc09e816de8050ffbc2bbfc 2024-11-15T16:50:46,888 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:40387/hbase/data/default/testSequentialEditLogSeqNum/5cebc9f67fc09e816de8050ffbc2bbfc doesn't exist for region: 5cebc9f67fc09e816de8050ffbc2bbfc on table testSequentialEditLogSeqNum 2024-11-15T16:50:46,888 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 5cebc9f67fc09e816de8050ffbc2bbfc on table testSequentialEditLogSeqNum 2024-11-15T16:50:46,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741875_1052 (size=62) 2024-11-15T16:50:46,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741875_1052 (size=62) 2024-11-15T16:50:46,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741875_1052 (size=62) 2024-11-15T16:50:46,900 INFO [StoreOpener-5cebc9f67fc09e816de8050ffbc2bbfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5cebc9f67fc09e816de8050ffbc2bbfc 2024-11-15T16:50:46,901 INFO [StoreOpener-5cebc9f67fc09e816de8050ffbc2bbfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cebc9f67fc09e816de8050ffbc2bbfc columnFamilyName a 2024-11-15T16:50:46,902 DEBUG [StoreOpener-5cebc9f67fc09e816de8050ffbc2bbfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:46,902 INFO [StoreOpener-5cebc9f67fc09e816de8050ffbc2bbfc-1 {}] regionserver.HStore(327): Store=5cebc9f67fc09e816de8050ffbc2bbfc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:46,902 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5cebc9f67fc09e816de8050ffbc2bbfc 2024-11-15T16:50:46,903 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testSequentialEditLogSeqNum/5cebc9f67fc09e816de8050ffbc2bbfc 2024-11-15T16:50:46,903 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testSequentialEditLogSeqNum/5cebc9f67fc09e816de8050ffbc2bbfc 2024-11-15T16:50:46,904 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5cebc9f67fc09e816de8050ffbc2bbfc 2024-11-15T16:50:46,904 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5cebc9f67fc09e816de8050ffbc2bbfc 2024-11-15T16:50:46,906 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5cebc9f67fc09e816de8050ffbc2bbfc 2024-11-15T16:50:46,908 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testSequentialEditLogSeqNum/5cebc9f67fc09e816de8050ffbc2bbfc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:50:46,908 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5cebc9f67fc09e816de8050ffbc2bbfc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59109886, jitterRate=-0.11919406056404114}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T16:50:46,909 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5cebc9f67fc09e816de8050ffbc2bbfc: Writing region info on filesystem at 1731689446887Initializing all the Stores at 1731689446899 (+12 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689446899Cleaning up temporary data from old regions at 1731689446904 (+5 ms)Region opened successfully at 1731689446909 (+5 ms) 2024-11-15T16:50:46,923 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5cebc9f67fc09e816de8050ffbc2bbfc 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-15T16:50:46,954 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testSequentialEditLogSeqNum/5cebc9f67fc09e816de8050ffbc2bbfc/.tmp/a/83d5266b105e4ec68f50d375a0916d9f is 81, key is testSequentialEditLogSeqNum/a:x0/1731689446909/Put/seqid=0 2024-11-15T16:50:46,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741876_1053 (size=5833) 2024-11-15T16:50:46,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741876_1053 (size=5833) 2024-11-15T16:50:46,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741876_1053 (size=5833) 2024-11-15T16:50:46,962 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testSequentialEditLogSeqNum/5cebc9f67fc09e816de8050ffbc2bbfc/.tmp/a/83d5266b105e4ec68f50d375a0916d9f 2024-11-15T16:50:46,971 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testSequentialEditLogSeqNum/5cebc9f67fc09e816de8050ffbc2bbfc/.tmp/a/83d5266b105e4ec68f50d375a0916d9f as hdfs://localhost:40387/hbase/data/default/testSequentialEditLogSeqNum/5cebc9f67fc09e816de8050ffbc2bbfc/a/83d5266b105e4ec68f50d375a0916d9f 2024-11-15T16:50:46,978 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testSequentialEditLogSeqNum/5cebc9f67fc09e816de8050ffbc2bbfc/a/83d5266b105e4ec68f50d375a0916d9f, entries=10, sequenceid=13, filesize=5.7 K 2024-11-15T16:50:46,980 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 5cebc9f67fc09e816de8050ffbc2bbfc in 57ms, sequenceid=13, compaction requested=false 2024-11-15T16:50:46,980 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5cebc9f67fc09e816de8050ffbc2bbfc: 2024-11-15T16:50:46,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-15T16:50:46,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-15T16:50:46,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-15T16:50:46,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-15T16:50:46,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-15T16:50:46,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741874_1051 (size=1844) 2024-11-15T16:50:46,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741874_1051 (size=1844) 2024-11-15T16:50:46,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741874_1051 (size=1844) 2024-11-15T16:50:47,008 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40387/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731689446860/wal.1731689446868, size=1.8 K (1844bytes) 2024-11-15T16:50:47,008 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40387/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731689446860/wal.1731689446868 2024-11-15T16:50:47,008 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40387/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731689446860/wal.1731689446868 after 0ms 2024-11-15T16:50:47,011 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731689446860/wal.1731689446868: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:47,011 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40387/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731689446860/wal.1731689446868 took 3ms 2024-11-15T16:50:47,014 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40387/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731689446860/wal.1731689446868 so closing down 2024-11-15T16:50:47,014 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T16:50:47,015 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731689446868.temp 2024-11-15T16:50:47,017 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testSequentialEditLogSeqNum/5cebc9f67fc09e816de8050ffbc2bbfc/recovered.edits/0000000000000000003-wal.1731689446868.temp 2024-11-15T16:50:47,017 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T16:50:47,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741877_1054 (size=1477) 2024-11-15T16:50:47,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741877_1054 (size=1477) 2024-11-15T16:50:47,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741877_1054 (size=1477) 2024-11-15T16:50:47,030 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testSequentialEditLogSeqNum/5cebc9f67fc09e816de8050ffbc2bbfc/recovered.edits/0000000000000000003-wal.1731689446868.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-15T16:50:47,031 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40387/hbase/data/default/testSequentialEditLogSeqNum/5cebc9f67fc09e816de8050ffbc2bbfc/recovered.edits/0000000000000000003-wal.1731689446868.temp to hdfs://localhost:40387/hbase/data/default/testSequentialEditLogSeqNum/5cebc9f67fc09e816de8050ffbc2bbfc/recovered.edits/0000000000000000020 2024-11-15T16:50:47,032 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 21 ms; skipped=2; WAL=hdfs://localhost:40387/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731689446860/wal.1731689446868, size=1.8 K, length=1844, corrupted=false, cancelled=false 2024-11-15T16:50:47,032 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40387/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731689446860/wal.1731689446868, journal: Splitting hdfs://localhost:40387/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731689446860/wal.1731689446868, size=1.8 K (1844bytes) at 1731689447008Finishing writing output for hdfs://localhost:40387/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731689446860/wal.1731689446868 so closing down at 1731689447014 (+6 ms)Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testSequentialEditLogSeqNum/5cebc9f67fc09e816de8050ffbc2bbfc/recovered.edits/0000000000000000003-wal.1731689446868.temp at 1731689447017 (+3 ms)3 split writer threads finished at 1731689447018 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testSequentialEditLogSeqNum/5cebc9f67fc09e816de8050ffbc2bbfc/recovered.edits/0000000000000000003-wal.1731689446868.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731689447030 (+12 ms)Rename recovered edits hdfs://localhost:40387/hbase/data/default/testSequentialEditLogSeqNum/5cebc9f67fc09e816de8050ffbc2bbfc/recovered.edits/0000000000000000003-wal.1731689446868.temp to hdfs://localhost:40387/hbase/data/default/testSequentialEditLogSeqNum/5cebc9f67fc09e816de8050ffbc2bbfc/recovered.edits/0000000000000000020 at 1731689447031 (+1 ms)Processed 17 edits across 1 Regions in 21 ms; skipped=2; WAL=hdfs://localhost:40387/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731689446860/wal.1731689446868, size=1.8 K, length=1844, corrupted=false, cancelled=false at 1731689447032 (+1 ms) 2024-11-15T16:50:47,054 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=411 (was 406) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:46750 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:53760 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:42180 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=272 (was 272), ProcessCount=11 (was 11), AvailableMemoryMB=5127 (was 5145) 2024-11-15T16:50:47,068 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=411, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=272, ProcessCount=11, AvailableMemoryMB=5125 2024-11-15T16:50:47,083 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:47,085 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:47,112 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:50:47,116 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-82121910, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/hregion-82121910, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:47,130 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-82121910/hregion-82121910.1731689447116, exclude list is [], retry=0 2024-11-15T16:50:47,134 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:47,134 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:47,135 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:47,138 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-82121910/hregion-82121910.1731689447116 2024-11-15T16:50:47,138 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:40741:40741)] 2024-11-15T16:50:47,139 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ab4b208c731814d5a327a2e25f4eefbb, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40387/hbase 2024-11-15T16:50:47,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741879_1056 (size=70) 2024-11-15T16:50:47,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741879_1056 (size=70) 2024-11-15T16:50:47,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741879_1056 (size=70) 2024-11-15T16:50:47,159 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:47,161 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,163 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab4b208c731814d5a327a2e25f4eefbb columnFamilyName a 2024-11-15T16:50:47,163 DEBUG [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:47,164 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(327): Store=ab4b208c731814d5a327a2e25f4eefbb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:47,164 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,166 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab4b208c731814d5a327a2e25f4eefbb columnFamilyName b 2024-11-15T16:50:47,166 DEBUG [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:47,167 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(327): Store=ab4b208c731814d5a327a2e25f4eefbb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:47,167 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,168 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab4b208c731814d5a327a2e25f4eefbb columnFamilyName c 2024-11-15T16:50:47,168 DEBUG [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:47,169 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(327): Store=ab4b208c731814d5a327a2e25f4eefbb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:47,169 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,170 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,171 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,172 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,172 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,172 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:50:47,174 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,177 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:50:47,177 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ab4b208c731814d5a327a2e25f4eefbb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66845686, jitterRate=-0.0039216578006744385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:50:47,179 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ab4b208c731814d5a327a2e25f4eefbb: Writing region info on filesystem at 1731689447159Initializing all the Stores at 1731689447161 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689447161Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689447161Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689447161Cleaning up temporary data from old regions at 1731689447172 (+11 ms)Region opened successfully at 1731689447179 (+7 ms) 2024-11-15T16:50:47,179 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ab4b208c731814d5a327a2e25f4eefbb, disabling compactions & flushes 2024-11-15T16:50:47,179 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb. 2024-11-15T16:50:47,179 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb. 2024-11-15T16:50:47,179 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb. after waiting 0 ms 2024-11-15T16:50:47,179 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb. 2024-11-15T16:50:47,180 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb. 2024-11-15T16:50:47,180 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ab4b208c731814d5a327a2e25f4eefbb: Waiting for close lock at 1731689447179Disabling compacts and flushes for region at 1731689447179Disabling writes for close at 1731689447179Writing region close event to WAL at 1731689447180 (+1 ms)Closed at 1731689447180 2024-11-15T16:50:47,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741878_1055 (size=95) 2024-11-15T16:50:47,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741878_1055 (size=95) 2024-11-15T16:50:47,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741878_1055 (size=95) 2024-11-15T16:50:47,187 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-15T16:50:47,187 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-82121910:(num 1731689447116) 2024-11-15T16:50:47,187 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:50:47,189 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:47,209 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190, exclude list is [], retry=0 2024-11-15T16:50:47,212 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:47,213 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:47,213 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:47,216 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190 2024-11-15T16:50:47,216 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:40741:40741)] 2024-11-15T16:50:47,217 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ab4b208c731814d5a327a2e25f4eefbb, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:47,217 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:47,217 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,217 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,219 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,220 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab4b208c731814d5a327a2e25f4eefbb columnFamilyName a 2024-11-15T16:50:47,220 DEBUG [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:47,221 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(327): Store=ab4b208c731814d5a327a2e25f4eefbb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:47,221 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,222 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab4b208c731814d5a327a2e25f4eefbb columnFamilyName b 2024-11-15T16:50:47,222 DEBUG [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:47,223 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(327): Store=ab4b208c731814d5a327a2e25f4eefbb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:47,223 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,224 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab4b208c731814d5a327a2e25f4eefbb columnFamilyName c 2024-11-15T16:50:47,224 DEBUG [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:47,224 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(327): Store=ab4b208c731814d5a327a2e25f4eefbb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:47,225 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,225 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,227 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,228 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,228 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,229 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:50:47,230 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:47,231 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ab4b208c731814d5a327a2e25f4eefbb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66075615, jitterRate=-0.015396609902381897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:50:47,232 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ab4b208c731814d5a327a2e25f4eefbb: Writing region info on filesystem at 1731689447217Initializing all the Stores at 1731689447218 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689447218Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689447219 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689447219Cleaning up temporary data from old regions at 1731689447228 (+9 ms)Region opened successfully at 1731689447232 (+4 ms) 2024-11-15T16:50:47,237 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731689447236/Put/seqid=0 2024-11-15T16:50:47,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741881_1058 (size=4826) 2024-11-15T16:50:47,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741881_1058 (size=4826) 2024-11-15T16:50:47,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741881_1058 (size=4826) 2024-11-15T16:50:47,246 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40387/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in ab4b208c731814d5a327a2e25f4eefbb/a 2024-11-15T16:50:47,256 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-15T16:50:47,256 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-15T16:50:47,257 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ab4b208c731814d5a327a2e25f4eefbb: 2024-11-15T16:50:47,258 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/a/9e868f6b6ffa4cf0b03975eed1f4e9e8_SeqId_3_ 2024-11-15T16:50:47,259 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40387/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into ab4b208c731814d5a327a2e25f4eefbb/a as hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/a/9e868f6b6ffa4cf0b03975eed1f4e9e8_SeqId_3_ - updating store file list. 2024-11-15T16:50:47,265 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9e868f6b6ffa4cf0b03975eed1f4e9e8_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-15T16:50:47,266 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/a/9e868f6b6ffa4cf0b03975eed1f4e9e8_SeqId_3_ into ab4b208c731814d5a327a2e25f4eefbb/a 2024-11-15T16:50:47,266 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40387/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into ab4b208c731814d5a327a2e25f4eefbb/a (new location: hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/a/9e868f6b6ffa4cf0b03975eed1f4e9e8_SeqId_3_) 2024-11-15T16:50:47,310 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40387/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190, size=0 (0bytes) 2024-11-15T16:50:47,310 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40387/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190 might be still open, length is 0 2024-11-15T16:50:47,310 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40387/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190 2024-11-15T16:50:47,310 WARN [IPC Server handler 4 on default port 40387 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-15T16:50:47,311 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40387/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190 after 1ms 2024-11-15T16:50:50,311 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:42382 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37733:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42382 dst: /127.0.0.1:37733 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37733 remote=/127.0.0.1:42382]. Total timeout mills is 60000, 56961 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:50:50,312 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:46926 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:42975:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46926 dst: /127.0.0.1:42975 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:50:50,312 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:53872 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39325:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53872 dst: /127.0.0.1:39325 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:50:50,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741880_1059 (size=474) 2024-11-15T16:50:50,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741880_1059 (size=474) 2024-11-15T16:50:50,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741880_1059 (size=474) 2024-11-15T16:50:51,312 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40387/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190 after 4001ms 2024-11-15T16:50:51,315 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:51,315 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40387/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190 took 4006ms 2024-11-15T16:50:51,317 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40387/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190; continuing. 2024-11-15T16:50:51,317 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40387/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190 so closing down 2024-11-15T16:50:51,317 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T16:50:51,319 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731689447190.temp 2024-11-15T16:50:51,321 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/recovered.edits/0000000000000000005-wal.1731689447190.temp 2024-11-15T16:50:51,321 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T16:50:51,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741882_1060 (size=259) 2024-11-15T16:50:51,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741882_1060 (size=259) 2024-11-15T16:50:51,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741882_1060 (size=259) 2024-11-15T16:50:51,331 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/recovered.edits/0000000000000000005-wal.1731689447190.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-15T16:50:51,332 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/recovered.edits/0000000000000000005-wal.1731689447190.temp to hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/recovered.edits/0000000000000000005 2024-11-15T16:50:51,332 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 16 ms; skipped=1; WAL=hdfs://localhost:40387/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190, size=0, length=0, corrupted=false, cancelled=false 2024-11-15T16:50:51,332 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40387/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190, journal: Splitting hdfs://localhost:40387/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190, size=0 (0bytes) at 1731689447310Finishing writing output for hdfs://localhost:40387/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190 so closing down at 1731689451317 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/recovered.edits/0000000000000000005-wal.1731689447190.temp at 1731689451321 (+4 ms)3 split writer threads finished at 1731689451321Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/recovered.edits/0000000000000000005-wal.1731689447190.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731689451331 (+10 ms)Rename recovered edits hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/recovered.edits/0000000000000000005-wal.1731689447190.temp to hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/recovered.edits/0000000000000000005 at 1731689451332 (+1 ms)Processed 2 edits across 1 Regions in 16 ms; skipped=1; WAL=hdfs://localhost:40387/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190, size=0, length=0, corrupted=false, cancelled=false at 1731689451332 2024-11-15T16:50:51,334 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40387/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190 to hdfs://localhost:40387/hbase/oldWALs/wal.1731689447190 2024-11-15T16:50:51,335 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/recovered.edits/0000000000000000005 2024-11-15T16:50:51,335 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:50:51,337 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:51,350 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689451338, exclude list is [], retry=0 2024-11-15T16:50:51,353 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:51,353 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:51,354 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:51,355 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689451338 2024-11-15T16:50:51,356 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:36277:36277)] 2024-11-15T16:50:51,356 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ab4b208c731814d5a327a2e25f4eefbb, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:51,356 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:51,356 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:51,356 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:51,358 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:51,359 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab4b208c731814d5a327a2e25f4eefbb columnFamilyName a 2024-11-15T16:50:51,359 DEBUG [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:51,365 DEBUG [StoreFileOpener-ab4b208c731814d5a327a2e25f4eefbb-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9e868f6b6ffa4cf0b03975eed1f4e9e8_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-15T16:50:51,366 DEBUG [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/a/9e868f6b6ffa4cf0b03975eed1f4e9e8_SeqId_3_ 2024-11-15T16:50:51,366 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(327): Store=ab4b208c731814d5a327a2e25f4eefbb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:51,366 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:51,367 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab4b208c731814d5a327a2e25f4eefbb columnFamilyName b 2024-11-15T16:50:51,367 DEBUG [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:51,368 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(327): Store=ab4b208c731814d5a327a2e25f4eefbb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:51,368 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:51,369 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab4b208c731814d5a327a2e25f4eefbb columnFamilyName c 2024-11-15T16:50:51,369 DEBUG [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:51,369 INFO [StoreOpener-ab4b208c731814d5a327a2e25f4eefbb-1 {}] regionserver.HStore(327): Store=ab4b208c731814d5a327a2e25f4eefbb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:51,370 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:51,371 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:51,372 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:51,373 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/recovered.edits/0000000000000000005 2024-11-15T16:50:51,375 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:51,376 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/recovered.edits/0000000000000000005 2024-11-15T16:50:51,376 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ab4b208c731814d5a327a2e25f4eefbb 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-15T16:50:51,393 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/.tmp/a/c1158dc91ac84958a520f26501901169 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731689447271/Put/seqid=0 2024-11-15T16:50:51,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741884_1062 (size=5149) 2024-11-15T16:50:51,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741884_1062 (size=5149) 2024-11-15T16:50:51,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741884_1062 (size=5149) 2024-11-15T16:50:51,400 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/.tmp/a/c1158dc91ac84958a520f26501901169 2024-11-15T16:50:51,406 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/.tmp/a/c1158dc91ac84958a520f26501901169 as hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/a/c1158dc91ac84958a520f26501901169 2024-11-15T16:50:51,412 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/a/c1158dc91ac84958a520f26501901169, entries=1, sequenceid=5, filesize=5.0 K 2024-11-15T16:50:51,412 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for ab4b208c731814d5a327a2e25f4eefbb in 36ms, sequenceid=5, compaction requested=false; wal=null 2024-11-15T16:50:51,413 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/recovered.edits/0000000000000000005 2024-11-15T16:50:51,415 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:51,415 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:51,415 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:50:51,417 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ab4b208c731814d5a327a2e25f4eefbb 2024-11-15T16:50:51,420 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab4b208c731814d5a327a2e25f4eefbb/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-15T16:50:51,421 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ab4b208c731814d5a327a2e25f4eefbb; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63825745, jitterRate=-0.04892228543758392}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:50:51,421 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ab4b208c731814d5a327a2e25f4eefbb: Writing region info on filesystem at 1731689451356Initializing all the Stores at 1731689451357 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689451357Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689451357Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689451358 (+1 ms)Obtaining lock to block concurrent updates at 1731689451376 (+18 ms)Preparing flush snapshotting stores in ab4b208c731814d5a327a2e25f4eefbb at 1731689451376Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731689451376Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb. at 1731689451376Flushing ab4b208c731814d5a327a2e25f4eefbb/a: creating writer at 1731689451377 (+1 ms)Flushing ab4b208c731814d5a327a2e25f4eefbb/a: appending metadata at 1731689451392 (+15 ms)Flushing ab4b208c731814d5a327a2e25f4eefbb/a: closing flushed file at 1731689451392Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e72800e: reopening flushed file at 1731689451406 (+14 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for ab4b208c731814d5a327a2e25f4eefbb in 36ms, sequenceid=5, compaction requested=false; wal=null at 1731689451413 (+7 ms)Cleaning up temporary data from old regions at 1731689451415 (+2 ms)Region opened successfully at 1731689451421 (+6 ms) 2024-11-15T16:50:51,425 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ab4b208c731814d5a327a2e25f4eefbb, disabling compactions & flushes 2024-11-15T16:50:51,425 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb. 2024-11-15T16:50:51,425 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb. 2024-11-15T16:50:51,425 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb. after waiting 0 ms 2024-11-15T16:50:51,425 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb. 2024-11-15T16:50:51,426 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731689447084.ab4b208c731814d5a327a2e25f4eefbb. 2024-11-15T16:50:51,426 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ab4b208c731814d5a327a2e25f4eefbb: Waiting for close lock at 1731689451425Disabling compacts and flushes for region at 1731689451425Disabling writes for close at 1731689451425Writing region close event to WAL at 1731689451426 (+1 ms)Closed at 1731689451426 2024-11-15T16:50:51,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741883_1061 (size=95) 2024-11-15T16:50:51,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741883_1061 (size=95) 2024-11-15T16:50:51,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741883_1061 (size=95) 2024-11-15T16:50:51,431 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-15T16:50:51,431 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731689451338) 2024-11-15T16:50:51,444 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=415 (was 411) Potentially hanging thread: IPC Client (1396088596) connection to localhost/127.0.0.1:44899 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_999242421_22 at /127.0.0.1:45158 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1396088596) connection to localhost/127.0.0.1:40387 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:40387 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_999242421_22 at /127.0.0.1:48088 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44899 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1024 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=250 (was 272), ProcessCount=11 (was 11), AvailableMemoryMB=5128 (was 5125) - AvailableMemoryMB LEAK? - 2024-11-15T16:50:51,457 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=415, OpenFileDescriptor=1022, MaxFileDescriptor=1048576, SystemLoadAverage=250, ProcessCount=11, AvailableMemoryMB=5127 2024-11-15T16:50:51,472 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:51,478 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T16:50:51,482 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a9bde5b3a1b5,43823,1731689436438 2024-11-15T16:50:51,485 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5161e510 2024-11-15T16:50:51,486 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T16:50:51,487 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T16:50:51,491 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T16:50:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-15T16:50:51,499 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T16:50:51,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-15T16:50:51,501 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:51,503 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T16:50:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T16:50:51,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741885_1063 (size=694) 2024-11-15T16:50:51,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741885_1063 (size=694) 2024-11-15T16:50:51,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741885_1063 (size=694) 2024-11-15T16:50:51,516 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bd339ac2e8d42b7857d656a05e282eea, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd 2024-11-15T16:50:51,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741886_1064 (size=77) 2024-11-15T16:50:51,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741886_1064 (size=77) 2024-11-15T16:50:51,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741886_1064 (size=77) 2024-11-15T16:50:51,528 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:51,528 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing bd339ac2e8d42b7857d656a05e282eea, disabling compactions & flushes 2024-11-15T16:50:51,528 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:51,528 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:51,528 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. after waiting 0 ms 2024-11-15T16:50:51,528 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:51,528 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:51,528 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for bd339ac2e8d42b7857d656a05e282eea: Waiting for close lock at 1731689451528Disabling compacts and flushes for region at 1731689451528Disabling writes for close at 1731689451528Writing region close event to WAL at 1731689451528Closed at 1731689451528 2024-11-15T16:50:51,530 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T16:50:51,537 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731689451530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731689451530"}]},"ts":"1731689451530"} 2024-11-15T16:50:51,540 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T16:50:51,542 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T16:50:51,545 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689451542"}]},"ts":"1731689451542"} 2024-11-15T16:50:51,549 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-15T16:50:51,549 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {a9bde5b3a1b5=0} racks are {/default-rack=0} 2024-11-15T16:50:51,550 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T16:50:51,550 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T16:50:51,551 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-15T16:50:51,551 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T16:50:51,551 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T16:50:51,551 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-15T16:50:51,551 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T16:50:51,551 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T16:50:51,551 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-15T16:50:51,551 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T16:50:51,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, ASSIGN}] 2024-11-15T16:50:51,554 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, ASSIGN 2024-11-15T16:50:51,555 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, ASSIGN; state=OFFLINE, location=a9bde5b3a1b5,38139,1731689437129; forceNewPlan=false, retain=false 2024-11-15T16:50:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T16:50:51,707 INFO [a9bde5b3a1b5:43823 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-15T16:50:51,708 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bd339ac2e8d42b7857d656a05e282eea, regionState=OPENING, regionLocation=a9bde5b3a1b5,38139,1731689437129 2024-11-15T16:50:51,711 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, ASSIGN because future has completed 2024-11-15T16:50:51,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,38139,1731689437129}] 2024-11-15T16:50:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T16:50:51,871 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:51,871 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => bd339ac2e8d42b7857d656a05e282eea, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:51,871 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:51,871 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:51,871 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:51,871 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:51,873 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:51,875 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd339ac2e8d42b7857d656a05e282eea columnFamilyName cf1 2024-11-15T16:50:51,875 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:51,875 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(327): Store=bd339ac2e8d42b7857d656a05e282eea/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:51,875 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:51,877 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd339ac2e8d42b7857d656a05e282eea columnFamilyName cf2 2024-11-15T16:50:51,877 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:51,877 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(327): Store=bd339ac2e8d42b7857d656a05e282eea/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:51,877 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:51,878 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:51,879 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:51,879 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:51,879 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:51,880 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-15T16:50:51,882 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:51,884 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:50:51,884 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened bd339ac2e8d42b7857d656a05e282eea; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63827141, jitterRate=-0.04890148341655731}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-15T16:50:51,885 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:51,885 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for bd339ac2e8d42b7857d656a05e282eea: Running coprocessor pre-open hook at 1731689451872Writing region info on filesystem at 1731689451872Initializing all the Stores at 1731689451873 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689451873Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689451873Cleaning up temporary data from old regions at 1731689451879 (+6 ms)Running coprocessor post-open hooks at 1731689451885 (+6 ms)Region opened successfully at 1731689451885 2024-11-15T16:50:51,887 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., pid=6, masterSystemTime=1731689451865 2024-11-15T16:50:51,890 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:51,890 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:51,891 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bd339ac2e8d42b7857d656a05e282eea, regionState=OPEN, openSeqNum=2, regionLocation=a9bde5b3a1b5,38139,1731689437129 2024-11-15T16:50:51,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,38139,1731689437129 because future has completed 2024-11-15T16:50:51,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T16:50:51,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,38139,1731689437129 in 184 msec 2024-11-15T16:50:51,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T16:50:51,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, ASSIGN in 347 msec 2024-11-15T16:50:51,904 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T16:50:51,905 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689451905"}]},"ts":"1731689451905"} 2024-11-15T16:50:51,907 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-15T16:50:51,909 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T16:50:51,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 416 msec 2024-11-15T16:50:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T16:50:52,136 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-15T16:50:52,136 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-15T16:50:52,138 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T16:50:52,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-15T16:50:52,144 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T16:50:52,145 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-15T16:50:52,157 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=2] 2024-11-15T16:50:52,172 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=bd339ac2e8d42b7857d656a05e282eea, source=a9bde5b3a1b5,38139,1731689437129, destination=a9bde5b3a1b5,45441,1731689437230, warming up region on a9bde5b3a1b5,45441,1731689437230 2024-11-15T16:50:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T16:50:52,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=bd339ac2e8d42b7857d656a05e282eea, source=a9bde5b3a1b5,38139,1731689437129, destination=a9bde5b3a1b5,45441,1731689437230, running balancer 2024-11-15T16:50:52,176 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58401, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T16:50:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, REOPEN/MOVE 2024-11-15T16:50:52,176 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, REOPEN/MOVE 2024-11-15T16:50:52,179 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=bd339ac2e8d42b7857d656a05e282eea, regionState=CLOSING, regionLocation=a9bde5b3a1b5,38139,1731689437129 2024-11-15T16:50:52,182 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45441 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45441 {}] regionserver.HRegion(7855): Warmup {ENCODED => bd339ac2e8d42b7857d656a05e282eea, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:52,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, REOPEN/MOVE because future has completed 2024-11-15T16:50:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45441 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:52,182 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T16:50:52,183 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,38139,1731689437129}] 2024-11-15T16:50:52,185 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:52,187 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd339ac2e8d42b7857d656a05e282eea columnFamilyName cf1 2024-11-15T16:50:52,187 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:52,188 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(327): Store=bd339ac2e8d42b7857d656a05e282eea/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:52,188 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:52,189 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd339ac2e8d42b7857d656a05e282eea columnFamilyName cf2 2024-11-15T16:50:52,189 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:52,190 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(327): Store=bd339ac2e8d42b7857d656a05e282eea/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45441 {}] regionserver.HRegion(1722): Closing bd339ac2e8d42b7857d656a05e282eea, disabling compactions & flushes 2024-11-15T16:50:52,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45441 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45441 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45441 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. after waiting 0 ms 2024-11-15T16:50:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45441 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:52,191 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45441 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45441 {}] regionserver.HRegion(1676): Region close journal for bd339ac2e8d42b7857d656a05e282eea: Waiting for close lock at 1731689452190Disabling compacts and flushes for region at 1731689452190Disabling writes for close at 1731689452190Writing region close event to WAL at 1731689452191 (+1 ms)Closed at 1731689452191 2024-11-15T16:50:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-15T16:50:52,341 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:52,341 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T16:50:52,342 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing bd339ac2e8d42b7857d656a05e282eea, disabling compactions & flushes 2024-11-15T16:50:52,342 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:52,342 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:52,342 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. after waiting 0 ms 2024-11-15T16:50:52,342 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:52,342 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing bd339ac2e8d42b7857d656a05e282eea 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-15T16:50:52,360 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/.tmp/cf1/577978ddccde4a83a136cc8497016ea9 is 35, key is r1/cf1:q/1731689452159/Put/seqid=0 2024-11-15T16:50:52,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741887_1065 (size=4783) 2024-11-15T16:50:52,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741887_1065 (size=4783) 2024-11-15T16:50:52,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741887_1065 (size=4783) 2024-11-15T16:50:52,367 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/.tmp/cf1/577978ddccde4a83a136cc8497016ea9 2024-11-15T16:50:52,374 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/.tmp/cf1/577978ddccde4a83a136cc8497016ea9 as hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/577978ddccde4a83a136cc8497016ea9 2024-11-15T16:50:52,380 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/577978ddccde4a83a136cc8497016ea9, entries=1, sequenceid=5, filesize=4.7 K 2024-11-15T16:50:52,381 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for bd339ac2e8d42b7857d656a05e282eea in 39ms, sequenceid=5, compaction requested=false 2024-11-15T16:50:52,381 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-15T16:50:52,387 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T16:50:52,390 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:52,390 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for bd339ac2e8d42b7857d656a05e282eea: Waiting for close lock at 1731689452342Running coprocessor pre-close hooks at 1731689452342Disabling compacts and flushes for region at 1731689452342Disabling writes for close at 1731689452342Obtaining lock to block concurrent updates at 1731689452342Preparing flush snapshotting stores in bd339ac2e8d42b7857d656a05e282eea at 1731689452342Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731689452342Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. at 1731689452343 (+1 ms)Flushing bd339ac2e8d42b7857d656a05e282eea/cf1: creating writer at 1731689452343Flushing bd339ac2e8d42b7857d656a05e282eea/cf1: appending metadata at 1731689452360 (+17 ms)Flushing bd339ac2e8d42b7857d656a05e282eea/cf1: closing flushed file at 1731689452360Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e3febe2: reopening flushed file at 1731689452374 (+14 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for bd339ac2e8d42b7857d656a05e282eea in 39ms, sequenceid=5, compaction requested=false at 1731689452381 (+7 ms)Writing region close event to WAL at 1731689452383 (+2 ms)Running coprocessor post-close hooks at 1731689452387 (+4 ms)Closed at 1731689452390 (+3 ms) 2024-11-15T16:50:52,390 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding bd339ac2e8d42b7857d656a05e282eea move to a9bde5b3a1b5,45441,1731689437230 record at close sequenceid=5 2024-11-15T16:50:52,393 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:52,394 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=bd339ac2e8d42b7857d656a05e282eea, regionState=CLOSED 2024-11-15T16:50:52,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,38139,1731689437129 because future has completed 2024-11-15T16:50:52,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T16:50:52,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,38139,1731689437129 in 215 msec 2024-11-15T16:50:52,403 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, REOPEN/MOVE; state=CLOSED, location=a9bde5b3a1b5,45441,1731689437230; forceNewPlan=false, retain=false 2024-11-15T16:50:52,553 INFO [a9bde5b3a1b5:43823 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-15T16:50:52,554 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=bd339ac2e8d42b7857d656a05e282eea, regionState=OPENING, regionLocation=a9bde5b3a1b5,45441,1731689437230 2024-11-15T16:50:52,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, REOPEN/MOVE because future has completed 2024-11-15T16:50:52,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,45441,1731689437230}] 2024-11-15T16:50:52,714 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:52,714 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => bd339ac2e8d42b7857d656a05e282eea, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:52,715 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:52,715 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:52,715 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:52,715 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:52,717 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:52,718 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd339ac2e8d42b7857d656a05e282eea columnFamilyName cf1 2024-11-15T16:50:52,718 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:52,726 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/577978ddccde4a83a136cc8497016ea9 2024-11-15T16:50:52,726 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(327): Store=bd339ac2e8d42b7857d656a05e282eea/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:52,726 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:52,727 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd339ac2e8d42b7857d656a05e282eea columnFamilyName cf2 2024-11-15T16:50:52,728 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:52,728 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(327): Store=bd339ac2e8d42b7857d656a05e282eea/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:52,728 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:52,729 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:52,731 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:52,731 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:52,731 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:52,732 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-15T16:50:52,734 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:52,735 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened bd339ac2e8d42b7857d656a05e282eea; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67413643, jitterRate=0.004541561007499695}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-15T16:50:52,735 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:52,736 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for bd339ac2e8d42b7857d656a05e282eea: Running coprocessor pre-open hook at 1731689452715Writing region info on filesystem at 1731689452715Initializing all the Stores at 1731689452717 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689452717Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689452717Cleaning up temporary data from old regions at 1731689452732 (+15 ms)Running coprocessor post-open hooks at 1731689452735 (+3 ms)Region opened successfully at 1731689452736 (+1 ms) 2024-11-15T16:50:52,737 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., pid=9, masterSystemTime=1731689452710 2024-11-15T16:50:52,740 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:52,741 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:52,741 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=bd339ac2e8d42b7857d656a05e282eea, regionState=OPEN, openSeqNum=9, regionLocation=a9bde5b3a1b5,45441,1731689437230 2024-11-15T16:50:52,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,45441,1731689437230 because future has completed 2024-11-15T16:50:52,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-15T16:50:52,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,45441,1731689437230 in 188 msec 2024-11-15T16:50:52,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, REOPEN/MOVE in 573 msec 2024-11-15T16:50:52,784 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T16:50:52,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T16:50:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:50286 deadline: 1731689512791, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a9bde5b3a1b5 port=45441 startCode=1731689437230. As of locationSeqNum=5. 2024-11-15T16:50:52,800 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a9bde5b3a1b5 port=45441 startCode=1731689437230. As of locationSeqNum=5. 2024-11-15T16:50:52,801 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a9bde5b3a1b5 port=45441 startCode=1731689437230. As of locationSeqNum=5. 2024-11-15T16:50:52,801 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,45441,1731689437230, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a9bde5b3a1b5 port=45441 startCode=1731689437230. As of locationSeqNum=5. 2024-11-15T16:50:52,915 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:50:52,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:50:52,928 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bd339ac2e8d42b7857d656a05e282eea 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-15T16:50:52,945 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/.tmp/cf1/aa1b3d3c3f424d1fa738ba900e7dfb0a is 29, key is r1/cf1:/1731689452919/DeleteFamily/seqid=0 2024-11-15T16:50:52,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741888_1066 (size=4906) 2024-11-15T16:50:52,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741888_1066 (size=4906) 2024-11-15T16:50:52,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741888_1066 (size=4906) 2024-11-15T16:50:52,954 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/.tmp/cf1/aa1b3d3c3f424d1fa738ba900e7dfb0a 2024-11-15T16:50:52,960 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aa1b3d3c3f424d1fa738ba900e7dfb0a 2024-11-15T16:50:52,974 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/.tmp/cf2/62f2670fc2f3407bbc5eb3b30adee0b7 is 29, key is r1/cf2:/1731689452919/DeleteFamily/seqid=0 2024-11-15T16:50:52,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741889_1067 (size=4906) 2024-11-15T16:50:52,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741889_1067 (size=4906) 2024-11-15T16:50:52,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741889_1067 (size=4906) 2024-11-15T16:50:52,983 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/.tmp/cf2/62f2670fc2f3407bbc5eb3b30adee0b7 2024-11-15T16:50:52,989 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 62f2670fc2f3407bbc5eb3b30adee0b7 2024-11-15T16:50:52,990 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/.tmp/cf1/aa1b3d3c3f424d1fa738ba900e7dfb0a as hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/aa1b3d3c3f424d1fa738ba900e7dfb0a 2024-11-15T16:50:52,997 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aa1b3d3c3f424d1fa738ba900e7dfb0a 2024-11-15T16:50:52,997 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/aa1b3d3c3f424d1fa738ba900e7dfb0a, entries=1, sequenceid=12, filesize=4.8 K 2024-11-15T16:50:52,998 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/.tmp/cf2/62f2670fc2f3407bbc5eb3b30adee0b7 as hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf2/62f2670fc2f3407bbc5eb3b30adee0b7 2024-11-15T16:50:53,003 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 62f2670fc2f3407bbc5eb3b30adee0b7 2024-11-15T16:50:53,004 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf2/62f2670fc2f3407bbc5eb3b30adee0b7, entries=1, sequenceid=12, filesize=4.8 K 2024-11-15T16:50:53,005 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for bd339ac2e8d42b7857d656a05e282eea in 78ms, sequenceid=12, compaction requested=false 2024-11-15T16:50:53,005 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bd339ac2e8d42b7857d656a05e282eea: 2024-11-15T16:50:53,007 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T16:50:53,009 DEBUG [Time-limited test {}] regionserver.HStore(1541): bd339ac2e8d42b7857d656a05e282eea/cf1 is initiating major compaction (all files) 2024-11-15T16:50:53,009 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T16:50:53,009 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:53,009 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of bd339ac2e8d42b7857d656a05e282eea/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,010 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/577978ddccde4a83a136cc8497016ea9, hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/aa1b3d3c3f424d1fa738ba900e7dfb0a] into tmpdir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/.tmp, totalSize=9.5 K 2024-11-15T16:50:53,011 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 577978ddccde4a83a136cc8497016ea9, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731689452159 2024-11-15T16:50:53,011 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting aa1b3d3c3f424d1fa738ba900e7dfb0a, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-15T16:50:53,021 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): bd339ac2e8d42b7857d656a05e282eea#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:50:53,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741890_1068 (size=4626) 2024-11-15T16:50:53,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741890_1068 (size=4626) 2024-11-15T16:50:53,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741890_1068 (size=4626) 2024-11-15T16:50:53,036 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/.tmp/cf1/f1865dcde7b6470baec6aed6ec60dd9c as hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/f1865dcde7b6470baec6aed6ec60dd9c 2024-11-15T16:50:53,050 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in bd339ac2e8d42b7857d656a05e282eea/cf1 of bd339ac2e8d42b7857d656a05e282eea into f1865dcde7b6470baec6aed6ec60dd9c(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:50:53,051 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for bd339ac2e8d42b7857d656a05e282eea: 2024-11-15T16:50:53,051 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-15T16:50:53,051 DEBUG [Time-limited test {}] regionserver.HStore(1541): bd339ac2e8d42b7857d656a05e282eea/cf2 is initiating major compaction (all files) 2024-11-15T16:50:53,051 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T16:50:53,051 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:50:53,051 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of bd339ac2e8d42b7857d656a05e282eea/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,052 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf2/62f2670fc2f3407bbc5eb3b30adee0b7] into tmpdir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/.tmp, totalSize=4.8 K 2024-11-15T16:50:53,052 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 62f2670fc2f3407bbc5eb3b30adee0b7, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-15T16:50:53,059 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): bd339ac2e8d42b7857d656a05e282eea#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:50:53,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741891_1069 (size=4592) 2024-11-15T16:50:53,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741891_1069 (size=4592) 2024-11-15T16:50:53,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741891_1069 (size=4592) 2024-11-15T16:50:53,078 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/.tmp/cf2/d41bce6c161a4bb59252a743a36623f0 as hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf2/d41bce6c161a4bb59252a743a36623f0 2024-11-15T16:50:53,087 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in bd339ac2e8d42b7857d656a05e282eea/cf2 of bd339ac2e8d42b7857d656a05e282eea into d41bce6c161a4bb59252a743a36623f0(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:50:53,087 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for bd339ac2e8d42b7857d656a05e282eea: 2024-11-15T16:50:53,092 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=bd339ac2e8d42b7857d656a05e282eea, source=a9bde5b3a1b5,45441,1731689437230, destination=a9bde5b3a1b5,38139,1731689437129, warming up region on a9bde5b3a1b5,38139,1731689437129 2024-11-15T16:50:53,093 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=bd339ac2e8d42b7857d656a05e282eea, source=a9bde5b3a1b5,45441,1731689437230, destination=a9bde5b3a1b5,38139,1731689437129, running balancer 2024-11-15T16:50:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, REOPEN/MOVE 2024-11-15T16:50:53,095 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, REOPEN/MOVE 2024-11-15T16:50:53,096 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=bd339ac2e8d42b7857d656a05e282eea, regionState=CLOSING, regionLocation=a9bde5b3a1b5,45441,1731689437230 2024-11-15T16:50:53,096 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(7855): Warmup {ENCODED => bd339ac2e8d42b7857d656a05e282eea, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:53,098 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, REOPEN/MOVE because future has completed 2024-11-15T16:50:53,099 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd339ac2e8d42b7857d656a05e282eea columnFamilyName cf1 2024-11-15T16:50:53,099 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:53,099 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T16:50:53,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,45441,1731689437230}] 2024-11-15T16:50:53,107 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/577978ddccde4a83a136cc8497016ea9 2024-11-15T16:50:53,111 INFO [StoreFileOpener-bd339ac2e8d42b7857d656a05e282eea-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aa1b3d3c3f424d1fa738ba900e7dfb0a 2024-11-15T16:50:53,112 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/aa1b3d3c3f424d1fa738ba900e7dfb0a 2024-11-15T16:50:53,116 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/f1865dcde7b6470baec6aed6ec60dd9c 2024-11-15T16:50:53,117 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(327): Store=bd339ac2e8d42b7857d656a05e282eea/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:53,117 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,118 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd339ac2e8d42b7857d656a05e282eea columnFamilyName cf2 2024-11-15T16:50:53,118 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:53,124 INFO [StoreFileOpener-bd339ac2e8d42b7857d656a05e282eea-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 62f2670fc2f3407bbc5eb3b30adee0b7 2024-11-15T16:50:53,124 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf2/62f2670fc2f3407bbc5eb3b30adee0b7 2024-11-15T16:50:53,130 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf2/d41bce6c161a4bb59252a743a36623f0 2024-11-15T16:50:53,130 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(327): Store=bd339ac2e8d42b7857d656a05e282eea/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(1722): Closing bd339ac2e8d42b7857d656a05e282eea, disabling compactions & flushes 2024-11-15T16:50:53,131 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. after waiting 0 ms 2024-11-15T16:50:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,132 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(1676): Region close journal for bd339ac2e8d42b7857d656a05e282eea: Waiting for close lock at 1731689453130Disabling compacts and flushes for region at 1731689453130Disabling writes for close at 1731689453131 (+1 ms)Writing region close event to WAL at 1731689453132 (+1 ms)Closed at 1731689453132 2024-11-15T16:50:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-15T16:50:53,254 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,254 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T16:50:53,254 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing bd339ac2e8d42b7857d656a05e282eea, disabling compactions & flushes 2024-11-15T16:50:53,254 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,254 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,254 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. after waiting 0 ms 2024-11-15T16:50:53,254 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,255 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/577978ddccde4a83a136cc8497016ea9, hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/aa1b3d3c3f424d1fa738ba900e7dfb0a] to archive 2024-11-15T16:50:53,258 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T16:50:53,262 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/577978ddccde4a83a136cc8497016ea9 to hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/577978ddccde4a83a136cc8497016ea9 2024-11-15T16:50:53,263 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/aa1b3d3c3f424d1fa738ba900e7dfb0a to hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/aa1b3d3c3f424d1fa738ba900e7dfb0a 2024-11-15T16:50:53,276 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf2/62f2670fc2f3407bbc5eb3b30adee0b7] to archive 2024-11-15T16:50:53,277 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T16:50:53,279 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf2/62f2670fc2f3407bbc5eb3b30adee0b7 to hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf2/62f2670fc2f3407bbc5eb3b30adee0b7 2024-11-15T16:50:53,284 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-15T16:50:53,285 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,285 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for bd339ac2e8d42b7857d656a05e282eea: Waiting for close lock at 1731689453254Running coprocessor pre-close hooks at 1731689453254Disabling compacts and flushes for region at 1731689453254Disabling writes for close at 1731689453254Writing region close event to WAL at 1731689453280 (+26 ms)Running coprocessor post-close hooks at 1731689453285 (+5 ms)Closed at 1731689453285 2024-11-15T16:50:53,285 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding bd339ac2e8d42b7857d656a05e282eea move to a9bde5b3a1b5,38139,1731689437129 record at close sequenceid=12 2024-11-15T16:50:53,288 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,289 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=bd339ac2e8d42b7857d656a05e282eea, regionState=CLOSED 2024-11-15T16:50:53,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,45441,1731689437230 because future has completed 2024-11-15T16:50:53,299 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-15T16:50:53,299 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,45441,1731689437230 in 197 msec 2024-11-15T16:50:53,300 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, REOPEN/MOVE; state=CLOSED, location=a9bde5b3a1b5,38139,1731689437129; forceNewPlan=false, retain=false 2024-11-15T16:50:53,450 INFO [a9bde5b3a1b5:43823 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-15T16:50:53,451 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=bd339ac2e8d42b7857d656a05e282eea, regionState=OPENING, regionLocation=a9bde5b3a1b5,38139,1731689437129 2024-11-15T16:50:53,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, REOPEN/MOVE because future has completed 2024-11-15T16:50:53,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,38139,1731689437129}] 2024-11-15T16:50:53,611 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,612 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => bd339ac2e8d42b7857d656a05e282eea, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:53,612 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,612 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:53,612 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,612 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,614 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,615 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd339ac2e8d42b7857d656a05e282eea columnFamilyName cf1 2024-11-15T16:50:53,615 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:53,622 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/f1865dcde7b6470baec6aed6ec60dd9c 2024-11-15T16:50:53,622 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(327): Store=bd339ac2e8d42b7857d656a05e282eea/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:53,622 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,623 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd339ac2e8d42b7857d656a05e282eea columnFamilyName cf2 2024-11-15T16:50:53,624 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:53,630 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf2/d41bce6c161a4bb59252a743a36623f0 2024-11-15T16:50:53,630 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(327): Store=bd339ac2e8d42b7857d656a05e282eea/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:53,630 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,631 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,632 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,633 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,633 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,634 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-15T16:50:53,636 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,637 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened bd339ac2e8d42b7857d656a05e282eea; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64476950, jitterRate=-0.03921857476234436}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-15T16:50:53,637 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,638 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for bd339ac2e8d42b7857d656a05e282eea: Running coprocessor pre-open hook at 1731689453612Writing region info on filesystem at 1731689453612Initializing all the Stores at 1731689453613 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689453614 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689453614Cleaning up temporary data from old regions at 1731689453633 (+19 ms)Running coprocessor post-open hooks at 1731689453637 (+4 ms)Region opened successfully at 1731689453638 (+1 ms) 2024-11-15T16:50:53,639 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., pid=12, masterSystemTime=1731689453607 2024-11-15T16:50:53,641 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,642 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,642 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=bd339ac2e8d42b7857d656a05e282eea, regionState=OPEN, openSeqNum=18, regionLocation=a9bde5b3a1b5,38139,1731689437129 2024-11-15T16:50:53,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,38139,1731689437129 because future has completed 2024-11-15T16:50:53,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-15T16:50:53,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,38139,1731689437129 in 193 msec 2024-11-15T16:50:53,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, REOPEN/MOVE in 557 msec 2024-11-15T16:50:53,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T16:50:53,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T16:50:53,700 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server a9bde5b3a1b5,38139,1731689437129: testing ***** 2024-11-15T16:50:53,701 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-15T16:50:53,703 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-15T16:50:53,704 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-15T16:50:53,706 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-15T16:50:53,707 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-15T16:50:53,715 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 365118976 }, "NonHeapMemoryUsage": { "committed": 171376640, "init": 7667712, "max": -1, "used": 166231720 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "a9bde5b3a1b5", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2068, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 13, "ProcessCallTime_mean": 5, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 7, "ProcessCallTime_75th_percentile": 10, "ProcessCallTime_90th_percentile": 11, "ProcessCallTime_95th_percentile": 12, "ProcessCallTime_98th_percentile": 12, "ProcessCallTime_99th_percentile": 12, "ProcessCallTime_99.9th_percentile": 12, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 13, "TotalCallTime_mean": 5, "TotalCallTime_25th_percentile": 4, "TotalCallTime_median": 7, "TotalCallTime_75th_percentile": 10, "TotalCallTime_90th_percentile": 11, "TotalCallTime_95th_percentile": 12, "TotalCallTime_98th_percentile": 12, "TotalCallTime_99th_percentile": 12, "TotalCallTime_99.9th_percentile": 12, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "a9bde5b3a1b5", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:50974", "tag.serverName": "a9bde5b3a1b5,38139,1731689437129", "tag.clusterId": "e3801ad5-a8ea-4892-a4a8-717494d68971", "tag.Context": "regionserver", "tag.Hostname": "a9bde5b3a1b5", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731689437129, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 199680, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 4, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 4, "ScanTime_min": 0, "ScanTime_max": 3, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 2, "ScanTime_90th_percentile": 2, "ScanTime_95th_percentile": 2, "ScanTime_98th_percentile": 2, "ScanTime_99th_percentile": 2, "ScanTime_99.9th_percentile": 2, "ScanTime_TimeRangeCount_0-1": 4, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 2, "Put_max": 8, "Put_mean": 5, "Put_25th_percentile": 3, "Put_median": 5, "Put_75th_percentile": 6, "Put_90th_percentile": 7, "Put_95th_percentile": 7, "Put_98th_percentile": 7, "Put_99th_percentile": 7, "Put_99.9th_percentile": 7, "Put_TimeRangeCount_0-1": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 2, "Get_max": 2, "Get_mean": 2, "Get_25th_percentile": 2, "Get_median": 2, "Get_75th_percentile": 2, "Get_90th_percentile": 2, "Get_95th_percentile": 2, "Get_98th_percentile": 2, "Get_99th_percentile": 2, "Get_99.9th_percentile": 2, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 4, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 72, "ScanSize_25th_percentile": 36, "ScanSize_median": 72, "ScanSize_75th_percentile": 108, "ScanSize_90th_percentile": 129, "ScanSize_95th_percentile": 136, "ScanSize_98th_percentile": 141, "ScanSize_99th_percentile": 142, "ScanSize_99.9th_percentile": 143, "ScanSize_SizeRangeCount_0-10": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-15T16:50:53,718 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43823 {}] master.MasterRpcServices(700): a9bde5b3a1b5,38139,1731689437129 reported a fatal error: ***** ABORTING region server a9bde5b3a1b5,38139,1731689437129: testing ***** 2024-11-15T16:50:53,720 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a9bde5b3a1b5,38139,1731689437129' ***** 2024-11-15T16:50:53,720 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-15T16:50:53,720 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T16:50:53,720 INFO [RS:0;a9bde5b3a1b5:38139 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-15T16:50:53,720 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T16:50:53,721 INFO [RS:0;a9bde5b3a1b5:38139 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-15T16:50:53,721 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(3091): Received CLOSE for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45441 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:53534 deadline: 1731689513721, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a9bde5b3a1b5 port=38139 startCode=1731689437129. As of locationSeqNum=12. 2024-11-15T16:50:53,721 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(956): aborting server a9bde5b3a1b5,38139,1731689437129 2024-11-15T16:50:53,721 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T16:50:53,722 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bd339ac2e8d42b7857d656a05e282eea, disabling compactions & flushes 2024-11-15T16:50:53,722 INFO [RS:0;a9bde5b3a1b5:38139 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a9bde5b3a1b5:38139. 2024-11-15T16:50:53,722 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,45441,1731689437230, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,45441,1731689437230, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a9bde5b3a1b5 port=38139 startCode=1731689437129. As of locationSeqNum=12. 2024-11-15T16:50:53,722 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,722 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,722 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,45441,1731689437230, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a9bde5b3a1b5 port=38139 startCode=1731689437129. As of locationSeqNum=12. 2024-11-15T16:50:53,722 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:50:53,722 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. after waiting 0 ms 2024-11-15T16:50:53,722 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,45441,1731689437230, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a9bde5b3a1b5 port=38139 startCode=1731689437129. As of locationSeqNum=12. 2024-11-15T16:50:53,722 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,722 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:50:53,723 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T16:50:53,723 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T16:50:53,723 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T16:50:53,723 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T16:50:53,723 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-15T16:50:53,723 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, bd339ac2e8d42b7857d656a05e282eea=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.} 2024-11-15T16:50:53,723 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T16:50:53,724 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T16:50:53,724 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T16:50:53,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a9bde5b3a1b5,38139,1731689437129 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:50:53,724 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T16:50:53,724 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T16:50:53,724 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:53,725 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a9bde5b3a1b5,38139,1731689437129 aborting 2024-11-15T16:50:53,725 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,725 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a9bde5b3a1b5,38139,1731689437129 aborting 2024-11-15T16:50:53,725 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1 from cache 2024-11-15T16:50:53,725 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bd339ac2e8d42b7857d656a05e282eea: Waiting for close lock at 1731689453721Running coprocessor pre-close hooks at 1731689453722 (+1 ms)Disabling compacts and flushes for region at 1731689453722Disabling writes for close at 1731689453722Writing region close event to WAL at 1731689453724 (+2 ms)Running coprocessor post-close hooks at 1731689453724Closed at 1731689453725 (+1 ms) 2024-11-15T16:50:53,725 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:53,726 ERROR [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-15T16:50:53,726 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T16:50:53,726 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T16:50:53,726 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731689453723Running coprocessor pre-close hooks at 1731689453723Disabling compacts and flushes for region at 1731689453723Disabling writes for close at 1731689453724 (+1 ms)Writing region close event to WAL at 1731689453726 (+2 ms)Running coprocessor post-close hooks at 1731689453726Closed at 1731689453726 2024-11-15T16:50:53,726 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T16:50:53,787 INFO [regionserver/a9bde5b3a1b5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T16:50:53,835 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:50:53,838 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1] 2024-11-15T16:50:53,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a9bde5b3a1b5,38139,1731689437129 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:50:53,840 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a9bde5b3a1b5,38139,1731689437129 aborting 2024-11-15T16:50:53,840 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a9bde5b3a1b5,38139,1731689437129 aborting 2024-11-15T16:50:53,840 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1 from cache 2024-11-15T16:50:53,924 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(976): stopping server a9bde5b3a1b5,38139,1731689437129; all regions closed. 2024-11-15T16:50:53,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741836_1012 (size=3561) 2024-11-15T16:50:53,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741836_1012 (size=3561) 2024-11-15T16:50:53,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741836_1012 (size=3561) 2024-11-15T16:50:53,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741835_1011 (size=1406) 2024-11-15T16:50:53,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741835_1011 (size=1406) 2024-11-15T16:50:53,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741835_1011 (size=1406) 2024-11-15T16:50:53,932 DEBUG [RS:0;a9bde5b3a1b5:38139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:50:53,932 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T16:50:53,932 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T16:50:53,932 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.ChoreService(370): Chore service for: regionserver/a9bde5b3a1b5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-15T16:50:53,932 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T16:50:53,932 INFO [regionserver/a9bde5b3a1b5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T16:50:53,933 INFO [RS:0;a9bde5b3a1b5:38139 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38139 2024-11-15T16:50:53,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a9bde5b3a1b5,38139,1731689437129 2024-11-15T16:50:53,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T16:50:53,940 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T16:50:53,941 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a9bde5b3a1b5,38139,1731689437129] 2024-11-15T16:50:53,944 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a9bde5b3a1b5,38139,1731689437129 already deleted, retry=false 2024-11-15T16:50:53,944 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of a9bde5b3a1b5,38139,1731689437129 on a9bde5b3a1b5,43823,1731689436438 2024-11-15T16:50:53,949 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure a9bde5b3a1b5,38139,1731689437129, splitWal=true, meta=true 2024-11-15T16:50:53,951 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for a9bde5b3a1b5,38139,1731689437129 (carryingMeta=true) a9bde5b3a1b5,38139,1731689437129/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@d26e87d[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-15T16:50:53,953 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure a9bde5b3a1b5,38139,1731689437129, splitWal=true, meta=true 2024-11-15T16:50:53,954 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure a9bde5b3a1b5,38139,1731689437129, splitWal=true, meta=true, isMeta: true 2024-11-15T16:50:53,956 DEBUG [PEWorker-5 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting 2024-11-15T16:50:53,957 INFO [PEWorker-5 {}] master.SplitWALManager(105): a9bde5b3a1b5,38139,1731689437129 WAL count=1, meta=true 2024-11-15T16:50:53,960 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta}] 2024-11-15T16:50:53,966 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=a9bde5b3a1b5,40715,1731689437267 2024-11-15T16:50:53,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta, worker=a9bde5b3a1b5,40715,1731689437267}] 2024-11-15T16:50:54,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:50:54,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x101acd7477a0001, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:50:54,043 INFO [RS:0;a9bde5b3a1b5:38139 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T16:50:54,044 INFO [RS:0;a9bde5b3a1b5:38139 {}] regionserver.HRegionServer(1031): Exiting; stopping=a9bde5b3a1b5,38139,1731689437129; zookeeper connection closed. 2024-11-15T16:50:54,044 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@38462315 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@38462315 2024-11-15T16:50:54,045 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:50:54,047 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1] 2024-11-15T16:50:54,049 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server a9bde5b3a1b5:38139 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: a9bde5b3a1b5/172.17.0.2:38139 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:50:54,049 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1, error=java.net.ConnectException: Call to address=a9bde5b3a1b5:38139 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: a9bde5b3a1b5/172.17.0.2:38139 2024-11-15T16:50:54,049 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-15T16:50:54,049 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1 from cache 2024-11-15T16:50:54,050 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address a9bde5b3a1b5:38139 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: a9bde5b3a1b5/172.17.0.2:38139 2024-11-15T16:50:54,132 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T16:50:54,133 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44525, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T16:50:54,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-15T16:50:54,154 INFO [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta, size=3.5 K (3561bytes) 2024-11-15T16:50:54,154 INFO [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta 2024-11-15T16:50:54,154 INFO [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta after 0ms 2024-11-15T16:50:54,157 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:54,158 INFO [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta took 5ms 2024-11-15T16:50:54,168 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-15T16:50:54,169 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta so closing down 2024-11-15T16:50:54,169 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T16:50:54,171 INFO [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta.temp 2024-11-15T16:50:54,172 INFO [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta.temp 2024-11-15T16:50:54,174 INFO [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T16:50:54,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741892_1070 (size=3346) 2024-11-15T16:50:54,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741892_1070 (size=3346) 2024-11-15T16:50:54,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741892_1070 (size=3346) 2024-11-15T16:50:54,188 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta.temp (wrote 15 edits, skipped 0 edits in 1 ms) 2024-11-15T16:50:54,190 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta.temp to hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-15T16:50:54,190 INFO [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 30 ms; skipped=1; WAL=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta, size=3.5 K, length=3561, corrupted=false, cancelled=false 2024-11-15T16:50:54,190 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta, journal: Splitting hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta, size=3.5 K (3561bytes) at 1731689454154Finishing writing output for hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta so closing down at 1731689454169 (+15 ms)Creating recovered edits writer path=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta.temp at 1731689454172 (+3 ms)3 split writer threads finished at 1731689454174 (+2 ms)Closed recovered edits writer path=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta.temp (wrote 15 edits, skipped 0 edits in 1 ms) at 1731689454188 (+14 ms)Rename recovered edits hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta.temp to hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1731689454190 (+2 ms)Processed 16 edits across 1 Regions in 30 ms; skipped=1; WAL=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta, size=3.5 K, length=3561, corrupted=false, cancelled=false at 1731689454190 2024-11-15T16:50:54,190 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta 2024-11-15T16:50:54,192 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-15T16:50:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-15T16:50:54,204 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta to hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/oldWALs 2024-11-15T16:50:54,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-15T16:50:54,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta, worker=a9bde5b3a1b5,40715,1731689437267 in 235 msec 2024-11-15T16:50:54,210 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=a9bde5b3a1b5,40715,1731689437267 2024-11-15T16:50:54,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-15T16:50:54,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure a9bde5b3a1b5%2C38139%2C1731689437129.meta.1731689439081.meta, worker=a9bde5b3a1b5,40715,1731689437267 in 251 msec 2024-11-15T16:50:54,216 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting dir is empty, no logs to split. 2024-11-15T16:50:54,216 INFO [PEWorker-5 {}] master.SplitWALManager(105): a9bde5b3a1b5,38139,1731689437129 WAL count=0, meta=true 2024-11-15T16:50:54,216 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if a9bde5b3a1b5,38139,1731689437129 WAL splitting is done? wals=0, meta=true 2024-11-15T16:50:54,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T16:50:54,219 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T16:50:54,220 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T16:50:54,355 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:50:54,356 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1] 2024-11-15T16:50:54,357 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to a9bde5b3a1b5:38139 this server is in the failed servers list 2024-11-15T16:50:54,358 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=a9bde5b3a1b5:38139 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a9bde5b3a1b5:38139 2024-11-15T16:50:54,358 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a9bde5b3a1b5:38139 2024-11-15T16:50:54,358 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1 from cache 2024-11-15T16:50:54,370 DEBUG [a9bde5b3a1b5:43823 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-15T16:50:54,370 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(204): Hosts are {a9bde5b3a1b5=0} racks are {/default-rack=0} 2024-11-15T16:50:54,370 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T16:50:54,370 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T16:50:54,370 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T16:50:54,370 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T16:50:54,370 INFO [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T16:50:54,370 INFO [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T16:50:54,371 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T16:50:54,371 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a9bde5b3a1b5,40715,1731689437267 2024-11-15T16:50:54,372 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a9bde5b3a1b5,40715,1731689437267, state=OPENING 2024-11-15T16:50:54,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:50:54,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:50:54,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:50:54,374 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T16:50:54,374 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:50:54,375 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:50:54,375 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:50:54,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,40715,1731689437267}] 2024-11-15T16:50:54,531 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T16:50:54,531 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:54,532 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T16:50:54,533 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a9bde5b3a1b5%2C40715%2C1731689437267.meta, suffix=.meta, logDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,40715,1731689437267, archiveDir=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/oldWALs, maxLogs=32 2024-11-15T16:50:54,547 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,40715,1731689437267/a9bde5b3a1b5%2C40715%2C1731689437267.meta.1731689454534.meta, exclude list is [], retry=0 2024-11-15T16:50:54,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:54,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:54,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:54,552 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,40715,1731689437267/a9bde5b3a1b5%2C40715%2C1731689437267.meta.1731689454534.meta 2024-11-15T16:50:54,552 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:40741:40741)] 2024-11-15T16:50:54,552 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:54,553 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T16:50:54,553 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T16:50:54,553 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T16:50:54,553 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T16:50:54,553 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:54,553 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T16:50:54,553 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T16:50:54,555 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T16:50:54,556 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T16:50:54,556 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:54,556 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:50:54,557 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T16:50:54,557 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T16:50:54,557 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:54,558 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:50:54,558 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T16:50:54,559 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T16:50:54,559 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:54,559 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:50:54,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T16:50:54,560 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T16:50:54,560 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:54,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:50:54,561 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T16:50:54,562 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740 2024-11-15T16:50:54,563 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740 2024-11-15T16:50:54,564 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-15T16:50:54,566 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/recovered.edits/0000000000000000018: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:54,568 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-15T16:50:54,568 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-15T16:50:54,588 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/.tmp/info/d76f86007cf7475b9a0359b56c559a66 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea./info:regioninfo/1731689453642/Put/seqid=0 2024-11-15T16:50:54,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741894_1072 (size=11177) 2024-11-15T16:50:54,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741894_1072 (size=11177) 2024-11-15T16:50:54,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741894_1072 (size=11177) 2024-11-15T16:50:54,598 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/.tmp/info/d76f86007cf7475b9a0359b56c559a66 2024-11-15T16:50:54,619 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/.tmp/ns/e74c131d6f024117bdb952109be595f9 is 43, key is default/ns:d/1731689439280/Put/seqid=0 2024-11-15T16:50:54,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741895_1073 (size=5153) 2024-11-15T16:50:54,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741895_1073 (size=5153) 2024-11-15T16:50:54,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741895_1073 (size=5153) 2024-11-15T16:50:54,627 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/.tmp/ns/e74c131d6f024117bdb952109be595f9 2024-11-15T16:50:54,647 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/.tmp/table/2f0ec0c5f0fa441a8bb55463f938ef78 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731689451905/Put/seqid=0 2024-11-15T16:50:54,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741896_1074 (size=5431) 2024-11-15T16:50:54,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741896_1074 (size=5431) 2024-11-15T16:50:54,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741896_1074 (size=5431) 2024-11-15T16:50:54,654 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/.tmp/table/2f0ec0c5f0fa441a8bb55463f938ef78 2024-11-15T16:50:54,661 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/.tmp/info/d76f86007cf7475b9a0359b56c559a66 as hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/info/d76f86007cf7475b9a0359b56c559a66 2024-11-15T16:50:54,667 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/info/d76f86007cf7475b9a0359b56c559a66, entries=36, sequenceid=18, filesize=10.9 K 2024-11-15T16:50:54,668 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/.tmp/ns/e74c131d6f024117bdb952109be595f9 as hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/ns/e74c131d6f024117bdb952109be595f9 2024-11-15T16:50:54,673 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/ns/e74c131d6f024117bdb952109be595f9, entries=2, sequenceid=18, filesize=5.0 K 2024-11-15T16:50:54,674 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/.tmp/table/2f0ec0c5f0fa441a8bb55463f938ef78 as hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/table/2f0ec0c5f0fa441a8bb55463f938ef78 2024-11-15T16:50:54,680 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/table/2f0ec0c5f0fa441a8bb55463f938ef78, entries=2, sequenceid=18, filesize=5.3 K 2024-11-15T16:50:54,680 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 112ms, sequenceid=18, compaction requested=false; wal=null 2024-11-15T16:50:54,681 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-15T16:50:54,682 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T16:50:54,682 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T16:50:54,683 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T16:50:54,684 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T16:50:54,687 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-15T16:50:54,688 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72232260, jitterRate=0.0763445496559143}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T16:50:54,688 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T16:50:54,689 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731689454553Writing region info on filesystem at 1731689454553Initializing all the Stores at 1731689454554 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689454554Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689454554Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689454554Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689454555 (+1 ms)Obtaining lock to block concurrent updates at 1731689454568 (+13 ms)Preparing flush snapshotting stores in 1588230740 at 1731689454568Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1731689454569 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731689454569Flushing 1588230740/info: creating writer at 1731689454569Flushing 1588230740/info: appending metadata at 1731689454588 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731689454588Flushing 1588230740/ns: creating writer at 1731689454604 (+16 ms)Flushing 1588230740/ns: appending metadata at 1731689454619 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731689454619Flushing 1588230740/table: creating writer at 1731689454632 (+13 ms)Flushing 1588230740/table: appending metadata at 1731689454647 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731689454647Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c89c6ae: reopening flushed file at 1731689454660 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@98b5fa0: reopening flushed file at 1731689454667 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cf07952: reopening flushed file at 1731689454674 (+7 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 112ms, sequenceid=18, compaction requested=false; wal=null at 1731689454680 (+6 ms)Cleaning up temporary data from old regions at 1731689454682 (+2 ms)Running coprocessor post-open hooks at 1731689454688 (+6 ms)Region opened successfully at 1731689454689 (+1 ms) 2024-11-15T16:50:54,690 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1731689454527 2024-11-15T16:50:54,693 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T16:50:54,693 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T16:50:54,693 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=a9bde5b3a1b5,40715,1731689437267 2024-11-15T16:50:54,694 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a9bde5b3a1b5,40715,1731689437267, state=OPEN 2024-11-15T16:50:54,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:50:54,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:50:54,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:50:54,696 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,40715,1731689437267 2024-11-15T16:50:54,696 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:50:54,696 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:50:54,697 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:50:54,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-15T16:50:54,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,40715,1731689437267 in 322 msec 2024-11-15T16:50:54,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-15T16:50:54,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 482 msec 2024-11-15T16:50:54,703 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): a9bde5b3a1b5,38139,1731689437129 had 2 regions 2024-11-15T16:50:54,704 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure a9bde5b3a1b5,38139,1731689437129, splitWal=true, meta=true, isMeta: false 2024-11-15T16:50:54,706 INFO [PEWorker-2 {}] master.SplitWALManager(105): a9bde5b3a1b5,38139,1731689437129 WAL count=1, meta=false 2024-11-15T16:50:54,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721}] 2024-11-15T16:50:54,708 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=a9bde5b3a1b5,45441,1731689437230 2024-11-15T16:50:54,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721, worker=a9bde5b3a1b5,45441,1731689437230}] 2024-11-15T16:50:54,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45441 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-15T16:50:54,865 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:50:54,867 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,40715,1731689437267, seqNum=-1] 2024-11-15T16:50:54,867 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:50:54,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:50:54,879 INFO [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721, size=1.4 K (1406bytes) 2024-11-15T16:50:54,879 INFO [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721 2024-11-15T16:50:54,880 INFO [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721 after 1ms 2024-11-15T16:50:54,882 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:54,882 INFO [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721 took 3ms 2024-11-15T16:50:54,885 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for bd339ac2e8d42b7857d656a05e282eea: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-15T16:50:54,885 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721 so closing down 2024-11-15T16:50:54,886 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T16:50:54,886 INFO [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T16:50:54,886 INFO [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721, size=1.4 K, length=1406, corrupted=false, cancelled=false 2024-11-15T16:50:54,886 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721, journal: Splitting hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721, size=1.4 K (1406bytes) at 1731689454879Finishing writing output for hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721 so closing down at 1731689454886 (+7 ms)3 split writer threads finished at 1731689454886Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721, size=1.4 K, length=1406, corrupted=false, cancelled=false at 1731689454886 2024-11-15T16:50:54,886 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721 2024-11-15T16:50:54,886 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-15T16:50:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-15T16:50:54,890 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting/a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721 to hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/oldWALs 2024-11-15T16:50:54,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-15T16:50:54,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721, worker=a9bde5b3a1b5,45441,1731689437230 in 181 msec 2024-11-15T16:50:54,897 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=a9bde5b3a1b5,45441,1731689437230 2024-11-15T16:50:54,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-15T16:50:54,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure a9bde5b3a1b5%2C38139%2C1731689437129.1731689438721, worker=a9bde5b3a1b5,45441,1731689437230 in 191 msec 2024-11-15T16:50:54,901 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/WALs/a9bde5b3a1b5,38139,1731689437129-splitting dir is empty, no logs to split. 2024-11-15T16:50:54,901 INFO [PEWorker-2 {}] master.SplitWALManager(105): a9bde5b3a1b5,38139,1731689437129 WAL count=0, meta=false 2024-11-15T16:50:54,901 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if a9bde5b3a1b5,38139,1731689437129 WAL splitting is done? wals=0, meta=false 2024-11-15T16:50:54,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, ASSIGN}] 2024-11-15T16:50:54,904 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, ASSIGN 2024-11-15T16:50:54,905 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T16:50:54,985 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to a9bde5b3a1b5:38139 this server is in the failed servers list 2024-11-15T16:50:54,985 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=a9bde5b3a1b5:38139 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a9bde5b3a1b5:38139 2024-11-15T16:50:54,985 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a9bde5b3a1b5:38139 2024-11-15T16:50:54,985 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=12 from cache 2024-11-15T16:50:55,056 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(204): Hosts are {a9bde5b3a1b5=0} racks are {/default-rack=0} 2024-11-15T16:50:55,056 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T16:50:55,056 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T16:50:55,056 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T16:50:55,056 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T16:50:55,056 INFO [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T16:50:55,056 INFO [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T16:50:55,056 DEBUG [a9bde5b3a1b5:43823 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T16:50:55,057 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=bd339ac2e8d42b7857d656a05e282eea, regionState=OPENING, regionLocation=a9bde5b3a1b5,45441,1731689437230 2024-11-15T16:50:55,058 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server a9bde5b3a1b5:38139 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a9bde5b3a1b5/172.17.0.2:38139 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:50:55,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1, error=java.net.ConnectException: Call to address=a9bde5b3a1b5:38139 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a9bde5b3a1b5/172.17.0.2:38139 2024-11-15T16:50:55,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T16:50:55,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=-1 from cache 2024-11-15T16:50:55,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address a9bde5b3a1b5:38139 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a9bde5b3a1b5/172.17.0.2:38139 2024-11-15T16:50:55,165 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:50:55,165 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,40715,1731689437267, seqNum=-1] 2024-11-15T16:50:55,166 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:50:55,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46081, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:50:55,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, ASSIGN because future has completed 2024-11-15T16:50:55,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,45441,1731689437230}] 2024-11-15T16:50:55,201 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=18] 2024-11-15T16:50:55,202 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] ipc.AbstractRpcClient(357): Not trying to connect to a9bde5b3a1b5:38139 this server is in the failed servers list 2024-11-15T16:50:55,202 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=a9bde5b3a1b5:38139 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a9bde5b3a1b5:38139 2024-11-15T16:50:55,202 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a9bde5b3a1b5:38139 2024-11-15T16:50:55,202 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,38139,1731689437129, seqNum=18 from cache 2024-11-15T16:50:55,327 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:55,327 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => bd339ac2e8d42b7857d656a05e282eea, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:55,328 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:55,328 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:55,328 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:55,328 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:55,329 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:55,330 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd339ac2e8d42b7857d656a05e282eea columnFamilyName cf1 2024-11-15T16:50:55,331 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:55,337 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf1/f1865dcde7b6470baec6aed6ec60dd9c 2024-11-15T16:50:55,337 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(327): Store=bd339ac2e8d42b7857d656a05e282eea/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:55,337 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:55,338 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd339ac2e8d42b7857d656a05e282eea columnFamilyName cf2 2024-11-15T16:50:55,338 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:55,343 DEBUG [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/cf2/d41bce6c161a4bb59252a743a36623f0 2024-11-15T16:50:55,344 INFO [StoreOpener-bd339ac2e8d42b7857d656a05e282eea-1 {}] regionserver.HStore(327): Store=bd339ac2e8d42b7857d656a05e282eea/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:55,344 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:55,344 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:55,346 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:55,346 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:55,346 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:55,347 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-15T16:50:55,348 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:55,349 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened bd339ac2e8d42b7857d656a05e282eea; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60677238, jitterRate=-0.09583869576454163}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-15T16:50:55,349 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:50:55,350 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for bd339ac2e8d42b7857d656a05e282eea: Running coprocessor pre-open hook at 1731689455328Writing region info on filesystem at 1731689455328Initializing all the Stores at 1731689455329 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689455329Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689455329Cleaning up temporary data from old regions at 1731689455346 (+17 ms)Running coprocessor post-open hooks at 1731689455349 (+3 ms)Region opened successfully at 1731689455350 (+1 ms) 2024-11-15T16:50:55,351 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., pid=21, masterSystemTime=1731689455323 2024-11-15T16:50:55,353 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:55,353 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:50:55,354 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=bd339ac2e8d42b7857d656a05e282eea, regionState=OPEN, openSeqNum=18, regionLocation=a9bde5b3a1b5,45441,1731689437230 2024-11-15T16:50:55,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,45441,1731689437230 because future has completed 2024-11-15T16:50:55,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-15T16:50:55,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure bd339ac2e8d42b7857d656a05e282eea, server=a9bde5b3a1b5,45441,1731689437230 in 187 msec 2024-11-15T16:50:55,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-15T16:50:55,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bd339ac2e8d42b7857d656a05e282eea, ASSIGN in 457 msec 2024-11-15T16:50:55,362 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server a9bde5b3a1b5,38139,1731689437129 after splitting done 2024-11-15T16:50:55,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure a9bde5b3a1b5,38139,1731689437129, splitWal=true, meta=true in 1.4180 sec 2024-11-15T16:50:55,510 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea., hostname=a9bde5b3a1b5,45441,1731689437230, seqNum=18] 2024-11-15T16:50:55,526 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=410 (was 415), OpenFileDescriptor=1050 (was 1022) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=230 (was 250), ProcessCount=11 (was 11), AvailableMemoryMB=5073 (was 5127) 2024-11-15T16:50:55,527 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-11-15T16:50:55,541 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=410, OpenFileDescriptor=1050, MaxFileDescriptor=1048576, SystemLoadAverage=230, ProcessCount=11, AvailableMemoryMB=5072 2024-11-15T16:50:55,541 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-11-15T16:50:55,557 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:55,559 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:55,559 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:50:55,562 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-67944555, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/hregion-67944555, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:55,575 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-67944555/hregion-67944555.1731689455562, exclude list is [], retry=0 2024-11-15T16:50:55,578 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:55,578 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:55,578 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:55,580 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-67944555/hregion-67944555.1731689455562 2024-11-15T16:50:55,580 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:37569:37569)] 2024-11-15T16:50:55,581 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ea780225a4e0df2abdc20a5462cda85d, NAME => 'testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40387/hbase 2024-11-15T16:50:55,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741898_1076 (size=67) 2024-11-15T16:50:55,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741898_1076 (size=67) 2024-11-15T16:50:55,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741898_1076 (size=67) 2024-11-15T16:50:55,589 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:55,591 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,592 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea780225a4e0df2abdc20a5462cda85d columnFamilyName a 2024-11-15T16:50:55,592 DEBUG [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:55,593 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(327): Store=ea780225a4e0df2abdc20a5462cda85d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:55,593 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,594 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea780225a4e0df2abdc20a5462cda85d columnFamilyName b 2024-11-15T16:50:55,594 DEBUG [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:55,595 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(327): Store=ea780225a4e0df2abdc20a5462cda85d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:55,595 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,596 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea780225a4e0df2abdc20a5462cda85d columnFamilyName c 2024-11-15T16:50:55,596 DEBUG [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:55,596 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(327): Store=ea780225a4e0df2abdc20a5462cda85d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:55,596 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,597 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,597 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,598 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,598 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,599 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:50:55,600 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,601 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:50:55,602 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea780225a4e0df2abdc20a5462cda85d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60527702, jitterRate=-0.09806695580482483}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:50:55,603 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea780225a4e0df2abdc20a5462cda85d: Writing region info on filesystem at 1731689455590Initializing all the Stores at 1731689455590Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689455590Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689455591 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689455591Cleaning up temporary data from old regions at 1731689455598 (+7 ms)Region opened successfully at 1731689455602 (+4 ms) 2024-11-15T16:50:55,603 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ea780225a4e0df2abdc20a5462cda85d, disabling compactions & flushes 2024-11-15T16:50:55,603 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d. 2024-11-15T16:50:55,603 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d. 2024-11-15T16:50:55,603 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d. after waiting 0 ms 2024-11-15T16:50:55,603 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d. 2024-11-15T16:50:55,603 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d. 2024-11-15T16:50:55,603 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ea780225a4e0df2abdc20a5462cda85d: Waiting for close lock at 1731689455603Disabling compacts and flushes for region at 1731689455603Disabling writes for close at 1731689455603Writing region close event to WAL at 1731689455603Closed at 1731689455603 2024-11-15T16:50:55,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741897_1075 (size=95) 2024-11-15T16:50:55,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741897_1075 (size=95) 2024-11-15T16:50:55,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741897_1075 (size=95) 2024-11-15T16:50:55,607 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-15T16:50:55,607 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-67944555:(num 1731689455562) 2024-11-15T16:50:55,608 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:50:55,609 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:55,622 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689455609, exclude list is [], retry=0 2024-11-15T16:50:55,624 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:55,625 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:55,625 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:55,627 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689455609 2024-11-15T16:50:55,627 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:37569:37569)] 2024-11-15T16:50:55,627 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ea780225a4e0df2abdc20a5462cda85d, NAME => 'testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:55,628 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:55,628 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,628 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,630 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,631 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea780225a4e0df2abdc20a5462cda85d columnFamilyName a 2024-11-15T16:50:55,631 DEBUG [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:55,631 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(327): Store=ea780225a4e0df2abdc20a5462cda85d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:55,631 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,632 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea780225a4e0df2abdc20a5462cda85d columnFamilyName b 2024-11-15T16:50:55,632 DEBUG [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:55,632 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(327): Store=ea780225a4e0df2abdc20a5462cda85d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:55,633 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,633 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea780225a4e0df2abdc20a5462cda85d columnFamilyName c 2024-11-15T16:50:55,633 DEBUG [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:55,634 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(327): Store=ea780225a4e0df2abdc20a5462cda85d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:55,634 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,635 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,636 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,637 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,637 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,637 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:50:55,638 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:55,639 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea780225a4e0df2abdc20a5462cda85d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70064575, jitterRate=0.04404352605342865}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:50:55,640 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea780225a4e0df2abdc20a5462cda85d: Writing region info on filesystem at 1731689455628Initializing all the Stores at 1731689455629 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689455629Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689455630 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689455630Cleaning up temporary data from old regions at 1731689455637 (+7 ms)Region opened successfully at 1731689455640 (+3 ms) 2024-11-15T16:50:55,663 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ea780225a4e0df2abdc20a5462cda85d 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-15T16:50:55,680 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/.tmp/a/9041165613e34ad7a8f25855bcb8ff26 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731689455640/Put/seqid=0 2024-11-15T16:50:55,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741900_1078 (size=5958) 2024-11-15T16:50:55,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741900_1078 (size=5958) 2024-11-15T16:50:55,688 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/.tmp/a/9041165613e34ad7a8f25855bcb8ff26 2024-11-15T16:50:55,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741900_1078 (size=5958) 2024-11-15T16:50:55,716 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/.tmp/b/4db17c1024a64735822c0aaf11750fec is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731689455648/Put/seqid=0 2024-11-15T16:50:55,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741901_1079 (size=5958) 2024-11-15T16:50:55,724 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/.tmp/b/4db17c1024a64735822c0aaf11750fec 2024-11-15T16:50:55,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741901_1079 (size=5958) 2024-11-15T16:50:55,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741901_1079 (size=5958) 2024-11-15T16:50:55,746 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/.tmp/c/368d0f1434dc45228d002e72c167bc73 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731689455655/Put/seqid=0 2024-11-15T16:50:55,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741902_1080 (size=5958) 2024-11-15T16:50:55,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741902_1080 (size=5958) 2024-11-15T16:50:55,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741902_1080 (size=5958) 2024-11-15T16:50:55,755 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/.tmp/c/368d0f1434dc45228d002e72c167bc73 2024-11-15T16:50:55,760 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/.tmp/a/9041165613e34ad7a8f25855bcb8ff26 as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/a/9041165613e34ad7a8f25855bcb8ff26 2024-11-15T16:50:55,765 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/a/9041165613e34ad7a8f25855bcb8ff26, entries=10, sequenceid=33, filesize=5.8 K 2024-11-15T16:50:55,766 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/.tmp/b/4db17c1024a64735822c0aaf11750fec as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/b/4db17c1024a64735822c0aaf11750fec 2024-11-15T16:50:55,771 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/b/4db17c1024a64735822c0aaf11750fec, entries=10, sequenceid=33, filesize=5.8 K 2024-11-15T16:50:55,772 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/.tmp/c/368d0f1434dc45228d002e72c167bc73 as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/c/368d0f1434dc45228d002e72c167bc73 2024-11-15T16:50:55,777 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/c/368d0f1434dc45228d002e72c167bc73, entries=10, sequenceid=33, filesize=5.8 K 2024-11-15T16:50:55,778 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for ea780225a4e0df2abdc20a5462cda85d in 115ms, sequenceid=33, compaction requested=false 2024-11-15T16:50:55,778 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ea780225a4e0df2abdc20a5462cda85d: 2024-11-15T16:50:55,779 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ea780225a4e0df2abdc20a5462cda85d, disabling compactions & flushes 2024-11-15T16:50:55,779 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d. 2024-11-15T16:50:55,779 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d. 2024-11-15T16:50:55,779 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d. after waiting 0 ms 2024-11-15T16:50:55,779 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d. 2024-11-15T16:50:55,780 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d. 2024-11-15T16:50:55,780 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ea780225a4e0df2abdc20a5462cda85d: Waiting for close lock at 1731689455779Disabling compacts and flushes for region at 1731689455779Disabling writes for close at 1731689455779Writing region close event to WAL at 1731689455780 (+1 ms)Closed at 1731689455780 2024-11-15T16:50:55,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741899_1077 (size=3382) 2024-11-15T16:50:55,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741899_1077 (size=3382) 2024-11-15T16:50:55,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741899_1077 (size=3382) 2024-11-15T16:50:55,788 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/b/4db17c1024a64735822c0aaf11750fec to hdfs://localhost:40387/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/b/4db17c1024a64735822c0aaf11750fec 2024-11-15T16:50:55,805 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40387/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689455609, size=3.3 K (3382bytes) 2024-11-15T16:50:55,805 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40387/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689455609 2024-11-15T16:50:55,805 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40387/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689455609 after 0ms 2024-11-15T16:50:55,808 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689455609: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:55,808 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40387/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689455609 took 4ms 2024-11-15T16:50:55,811 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40387/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689455609 so closing down 2024-11-15T16:50:55,811 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T16:50:55,812 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731689455609.temp 2024-11-15T16:50:55,813 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/recovered.edits/0000000000000000003-wal.1731689455609.temp 2024-11-15T16:50:55,814 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T16:50:55,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741903_1081 (size=2944) 2024-11-15T16:50:55,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741903_1081 (size=2944) 2024-11-15T16:50:55,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741903_1081 (size=2944) 2024-11-15T16:50:56,224 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/recovered.edits/0000000000000000003-wal.1731689455609.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-15T16:50:56,226 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/recovered.edits/0000000000000000003-wal.1731689455609.temp to hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/recovered.edits/0000000000000000032 2024-11-15T16:50:56,226 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 417 ms; skipped=2; WAL=hdfs://localhost:40387/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689455609, size=3.3 K, length=3382, corrupted=false, cancelled=false 2024-11-15T16:50:56,226 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40387/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689455609, journal: Splitting hdfs://localhost:40387/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689455609, size=3.3 K (3382bytes) at 1731689455805Finishing writing output for hdfs://localhost:40387/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689455609 so closing down at 1731689455811 (+6 ms)Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/recovered.edits/0000000000000000003-wal.1731689455609.temp at 1731689455814 (+3 ms)3 split writer threads finished at 1731689455814Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/recovered.edits/0000000000000000003-wal.1731689455609.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731689456224 (+410 ms)Rename recovered edits hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/recovered.edits/0000000000000000003-wal.1731689455609.temp to hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/recovered.edits/0000000000000000032 at 1731689456226 (+2 ms)Processed 32 edits across 1 Regions in 417 ms; skipped=2; WAL=hdfs://localhost:40387/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689455609, size=3.3 K, length=3382, corrupted=false, cancelled=false at 1731689456226 2024-11-15T16:50:56,228 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40387/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689455609 to hdfs://localhost:40387/hbase/oldWALs/wal.1731689455609 2024-11-15T16:50:56,229 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/recovered.edits/0000000000000000032 2024-11-15T16:50:56,229 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:50:56,230 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:56,244 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689456231, exclude list is [], retry=0 2024-11-15T16:50:56,246 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:56,247 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:56,247 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:56,249 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689456231 2024-11-15T16:50:56,250 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:40741:40741)] 2024-11-15T16:50:56,250 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ea780225a4e0df2abdc20a5462cda85d, NAME => 'testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:56,250 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:56,250 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:56,250 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:56,251 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:56,252 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea780225a4e0df2abdc20a5462cda85d columnFamilyName a 2024-11-15T16:50:56,252 DEBUG [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:56,257 DEBUG [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/a/9041165613e34ad7a8f25855bcb8ff26 2024-11-15T16:50:56,257 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(327): Store=ea780225a4e0df2abdc20a5462cda85d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:56,258 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:56,259 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea780225a4e0df2abdc20a5462cda85d columnFamilyName b 2024-11-15T16:50:56,259 DEBUG [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:56,259 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(327): Store=ea780225a4e0df2abdc20a5462cda85d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:56,259 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:56,260 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea780225a4e0df2abdc20a5462cda85d columnFamilyName c 2024-11-15T16:50:56,260 DEBUG [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:56,265 DEBUG [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/c/368d0f1434dc45228d002e72c167bc73 2024-11-15T16:50:56,265 INFO [StoreOpener-ea780225a4e0df2abdc20a5462cda85d-1 {}] regionserver.HStore(327): Store=ea780225a4e0df2abdc20a5462cda85d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:56,265 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:56,266 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:56,268 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:56,268 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/recovered.edits/0000000000000000032 2024-11-15T16:50:56,270 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:50:56,271 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/recovered.edits/0000000000000000032 2024-11-15T16:50:56,272 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ea780225a4e0df2abdc20a5462cda85d 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-15T16:50:56,293 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/.tmp/b/c0a526686ad44a3db3aa85e85652745b is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731689455648/Put/seqid=0 2024-11-15T16:50:56,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741905_1083 (size=5958) 2024-11-15T16:50:56,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741905_1083 (size=5958) 2024-11-15T16:50:56,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741905_1083 (size=5958) 2024-11-15T16:50:56,300 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/.tmp/b/c0a526686ad44a3db3aa85e85652745b 2024-11-15T16:50:56,306 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/.tmp/b/c0a526686ad44a3db3aa85e85652745b as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/b/c0a526686ad44a3db3aa85e85652745b 2024-11-15T16:50:56,311 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/b/c0a526686ad44a3db3aa85e85652745b, entries=10, sequenceid=32, filesize=5.8 K 2024-11-15T16:50:56,311 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for ea780225a4e0df2abdc20a5462cda85d in 39ms, sequenceid=32, compaction requested=false; wal=null 2024-11-15T16:50:56,312 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/recovered.edits/0000000000000000032 2024-11-15T16:50:56,313 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:56,313 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:56,314 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:50:56,315 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea780225a4e0df2abdc20a5462cda85d 2024-11-15T16:50:56,317 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/ea780225a4e0df2abdc20a5462cda85d/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-15T16:50:56,318 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea780225a4e0df2abdc20a5462cda85d; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71133701, jitterRate=0.05997474491596222}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:50:56,319 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea780225a4e0df2abdc20a5462cda85d: Writing region info on filesystem at 1731689456250Initializing all the Stores at 1731689456251 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689456251Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689456251Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689456251Obtaining lock to block concurrent updates at 1731689456272 (+21 ms)Preparing flush snapshotting stores in ea780225a4e0df2abdc20a5462cda85d at 1731689456272Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731689456272Flushing stores of testReplayEditsWrittenViaHRegion,,1731689455557.ea780225a4e0df2abdc20a5462cda85d. at 1731689456272Flushing ea780225a4e0df2abdc20a5462cda85d/b: creating writer at 1731689456272Flushing ea780225a4e0df2abdc20a5462cda85d/b: appending metadata at 1731689456292 (+20 ms)Flushing ea780225a4e0df2abdc20a5462cda85d/b: closing flushed file at 1731689456292Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68a57d7d: reopening flushed file at 1731689456306 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for ea780225a4e0df2abdc20a5462cda85d in 39ms, sequenceid=32, compaction requested=false; wal=null at 1731689456312 (+6 ms)Cleaning up temporary data from old regions at 1731689456313 (+1 ms)Region opened successfully at 1731689456319 (+6 ms) 2024-11-15T16:50:56,340 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=418 (was 410) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:41432 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:45158 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1263208986-172.17.0.2-1731689433335:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1263208986-172.17.0.2-1731689433335:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:48334 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1263208986-172.17.0.2-1731689433335:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:45324 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:48088 [Waiting for operation #44] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:41584 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1050) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=230 (was 230), ProcessCount=11 (was 11), AvailableMemoryMB=5062 (was 5072) 2024-11-15T16:50:56,340 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-15T16:50:56,353 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=418, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=230, ProcessCount=11, AvailableMemoryMB=5061 2024-11-15T16:50:56,353 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-15T16:50:56,368 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:56,370 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:50:56,371 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:50:56,373 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-90660768, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/hregion-90660768, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:56,387 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-90660768/hregion-90660768.1731689456374, exclude list is [], retry=0 2024-11-15T16:50:56,389 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:56,389 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:56,390 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:56,392 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-90660768/hregion-90660768.1731689456374 2024-11-15T16:50:56,392 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:40741:40741)] 2024-11-15T16:50:56,392 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e5381a957f04881a4eee5cd168aa7f50, NAME => 'testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40387/hbase 2024-11-15T16:50:56,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741907_1085 (size=68) 2024-11-15T16:50:56,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741907_1085 (size=68) 2024-11-15T16:50:56,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741907_1085 (size=68) 2024-11-15T16:50:56,401 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:56,402 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,403 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5381a957f04881a4eee5cd168aa7f50 columnFamilyName a 2024-11-15T16:50:56,403 DEBUG [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:56,404 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(327): Store=e5381a957f04881a4eee5cd168aa7f50/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:56,404 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,405 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5381a957f04881a4eee5cd168aa7f50 columnFamilyName b 2024-11-15T16:50:56,405 DEBUG [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:56,405 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(327): Store=e5381a957f04881a4eee5cd168aa7f50/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:56,405 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,406 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5381a957f04881a4eee5cd168aa7f50 columnFamilyName c 2024-11-15T16:50:56,406 DEBUG [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:56,407 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(327): Store=e5381a957f04881a4eee5cd168aa7f50/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:56,407 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,408 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,408 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,409 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,409 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,409 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:50:56,410 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,412 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:50:56,413 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e5381a957f04881a4eee5cd168aa7f50; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72430452, jitterRate=0.07929784059524536}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:50:56,413 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e5381a957f04881a4eee5cd168aa7f50: Writing region info on filesystem at 1731689456401Initializing all the Stores at 1731689456401Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689456401Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689456402 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689456402Cleaning up temporary data from old regions at 1731689456409 (+7 ms)Region opened successfully at 1731689456413 (+4 ms) 2024-11-15T16:50:56,413 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e5381a957f04881a4eee5cd168aa7f50, disabling compactions & flushes 2024-11-15T16:50:56,413 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50. 2024-11-15T16:50:56,413 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50. 2024-11-15T16:50:56,413 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50. after waiting 0 ms 2024-11-15T16:50:56,413 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50. 2024-11-15T16:50:56,414 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50. 2024-11-15T16:50:56,414 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e5381a957f04881a4eee5cd168aa7f50: Waiting for close lock at 1731689456413Disabling compacts and flushes for region at 1731689456413Disabling writes for close at 1731689456413Writing region close event to WAL at 1731689456414 (+1 ms)Closed at 1731689456414 2024-11-15T16:50:56,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741906_1084 (size=95) 2024-11-15T16:50:56,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741906_1084 (size=95) 2024-11-15T16:50:56,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741906_1084 (size=95) 2024-11-15T16:50:56,418 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-15T16:50:56,418 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-90660768:(num 1731689456374) 2024-11-15T16:50:56,418 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:50:56,420 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:50:56,433 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689456420, exclude list is [], retry=0 2024-11-15T16:50:56,436 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:50:56,436 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:50:56,437 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:50:56,439 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689456420 2024-11-15T16:50:56,439 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:40741:40741)] 2024-11-15T16:50:56,492 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e5381a957f04881a4eee5cd168aa7f50, NAME => 'testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:50:56,494 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,494 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:50:56,494 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,495 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,497 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,498 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5381a957f04881a4eee5cd168aa7f50 columnFamilyName a 2024-11-15T16:50:56,498 DEBUG [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:56,499 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(327): Store=e5381a957f04881a4eee5cd168aa7f50/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:56,499 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,500 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5381a957f04881a4eee5cd168aa7f50 columnFamilyName b 2024-11-15T16:50:56,500 DEBUG [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:56,501 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(327): Store=e5381a957f04881a4eee5cd168aa7f50/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:56,501 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,502 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5381a957f04881a4eee5cd168aa7f50 columnFamilyName c 2024-11-15T16:50:56,502 DEBUG [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:50:56,503 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(327): Store=e5381a957f04881a4eee5cd168aa7f50/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:50:56,503 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,503 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,505 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,506 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,506 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,507 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:50:56,508 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,509 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e5381a957f04881a4eee5cd168aa7f50; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72017327, jitterRate=0.07314179837703705}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:50:56,509 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:50:56,511 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e5381a957f04881a4eee5cd168aa7f50: Running coprocessor pre-open hook at 1731689456495Writing region info on filesystem at 1731689456495Initializing all the Stores at 1731689456496 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689456496Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689456496Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689456496Cleaning up temporary data from old regions at 1731689456506 (+10 ms)Running coprocessor post-open hooks at 1731689456509 (+3 ms)Region opened successfully at 1731689456510 (+1 ms) 2024-11-15T16:50:56,525 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e5381a957f04881a4eee5cd168aa7f50 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-15T16:50:56,527 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e5381a957f04881a4eee5cd168aa7f50/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:50:56,862 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-15T16:50:56,862 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-15T16:50:56,863 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-15T16:50:56,863 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-15T16:50:56,864 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-15T16:50:56,864 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-15T16:50:56,865 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-15T16:50:56,865 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-15T16:50:56,866 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-15T16:50:56,866 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-15T16:50:57,398 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T16:50:57,528 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e5381a957f04881a4eee5cd168aa7f50/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:50:58,528 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e5381a957f04881a4eee5cd168aa7f50/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:50:59,529 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e5381a957f04881a4eee5cd168aa7f50/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:51:00,529 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e5381a957f04881a4eee5cd168aa7f50/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:51:01,530 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e5381a957f04881a4eee5cd168aa7f50/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:51:02,531 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e5381a957f04881a4eee5cd168aa7f50/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:51:03,531 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e5381a957f04881a4eee5cd168aa7f50/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:51:04,532 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e5381a957f04881a4eee5cd168aa7f50/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:51:05,533 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e5381a957f04881a4eee5cd168aa7f50/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:51:05,534 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e5381a957f04881a4eee5cd168aa7f50: 2024-11-15T16:51:05,534 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50. 2024-11-15T16:51:05,546 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e5381a957f04881a4eee5cd168aa7f50: 2024-11-15T16:51:05,546 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-15T16:51:05,546 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e5381a957f04881a4eee5cd168aa7f50, disabling compactions & flushes 2024-11-15T16:51:05,546 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50. 2024-11-15T16:51:05,546 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50. 2024-11-15T16:51:05,546 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50. after waiting 0 ms 2024-11-15T16:51:05,546 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50. 2024-11-15T16:51:05,546 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50. 2024-11-15T16:51:05,547 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50. 2024-11-15T16:51:05,547 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e5381a957f04881a4eee5cd168aa7f50: Waiting for close lock at 1731689465546Running coprocessor pre-close hooks at 1731689465546Disabling compacts and flushes for region at 1731689465546Disabling writes for close at 1731689465546Writing region close event to WAL at 1731689465546Running coprocessor post-close hooks at 1731689465547 (+1 ms)Closed at 1731689465547 2024-11-15T16:51:05,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741908_1086 (size=2685) 2024-11-15T16:51:05,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741908_1086 (size=2685) 2024-11-15T16:51:05,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741908_1086 (size=2685) 2024-11-15T16:51:05,566 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40387/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689456420, size=2.6 K (2685bytes) 2024-11-15T16:51:05,566 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40387/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689456420 2024-11-15T16:51:05,566 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40387/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689456420 after 0ms 2024-11-15T16:51:05,568 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689456420: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:51:05,569 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40387/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689456420 took 4ms 2024-11-15T16:51:05,571 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40387/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689456420 so closing down 2024-11-15T16:51:05,571 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T16:51:05,572 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731689456420.temp 2024-11-15T16:51:05,573 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/recovered.edits/0000000000000000004-wal.1731689456420.temp 2024-11-15T16:51:05,574 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T16:51:05,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741909_1087 (size=2094) 2024-11-15T16:51:05,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741909_1087 (size=2094) 2024-11-15T16:51:05,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741909_1087 (size=2094) 2024-11-15T16:51:05,581 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/recovered.edits/0000000000000000004-wal.1731689456420.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-15T16:51:05,583 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/recovered.edits/0000000000000000004-wal.1731689456420.temp to hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/recovered.edits/0000000000000000026 2024-11-15T16:51:05,583 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 14 ms; skipped=3; WAL=hdfs://localhost:40387/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689456420, size=2.6 K, length=2685, corrupted=false, cancelled=false 2024-11-15T16:51:05,583 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40387/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689456420, journal: Splitting hdfs://localhost:40387/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689456420, size=2.6 K (2685bytes) at 1731689465566Finishing writing output for hdfs://localhost:40387/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689456420 so closing down at 1731689465571 (+5 ms)Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/recovered.edits/0000000000000000004-wal.1731689456420.temp at 1731689465573 (+2 ms)3 split writer threads finished at 1731689465574 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/recovered.edits/0000000000000000004-wal.1731689456420.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731689465582 (+8 ms)Rename recovered edits hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/recovered.edits/0000000000000000004-wal.1731689456420.temp to hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/recovered.edits/0000000000000000026 at 1731689465583 (+1 ms)Processed 23 edits across 1 Regions in 14 ms; skipped=3; WAL=hdfs://localhost:40387/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689456420, size=2.6 K, length=2685, corrupted=false, cancelled=false at 1731689465583 2024-11-15T16:51:05,585 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40387/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689456420 to hdfs://localhost:40387/hbase/oldWALs/wal.1731689456420 2024-11-15T16:51:05,586 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/recovered.edits/0000000000000000026 2024-11-15T16:51:05,586 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:51:05,588 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:51:05,590 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T16:51:05,602 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689465588, exclude list is [], retry=0 2024-11-15T16:51:05,604 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:51:05,605 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:51:05,605 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:51:05,606 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689465588 2024-11-15T16:51:05,608 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:40741:40741)] 2024-11-15T16:51:05,608 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e5381a957f04881a4eee5cd168aa7f50, NAME => 'testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:51:05,609 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:51:05,609 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:51:05,609 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:51:05,609 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:51:05,611 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:51:05,612 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5381a957f04881a4eee5cd168aa7f50 columnFamilyName a 2024-11-15T16:51:05,612 DEBUG [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:05,613 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(327): Store=e5381a957f04881a4eee5cd168aa7f50/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:05,613 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:51:05,614 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5381a957f04881a4eee5cd168aa7f50 columnFamilyName b 2024-11-15T16:51:05,614 DEBUG [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:05,614 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(327): Store=e5381a957f04881a4eee5cd168aa7f50/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:05,615 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:51:05,615 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5381a957f04881a4eee5cd168aa7f50 columnFamilyName c 2024-11-15T16:51:05,615 DEBUG [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:05,616 INFO [StoreOpener-e5381a957f04881a4eee5cd168aa7f50-1 {}] regionserver.HStore(327): Store=e5381a957f04881a4eee5cd168aa7f50/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:05,616 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:51:05,616 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:51:05,618 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:51:05,618 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/recovered.edits/0000000000000000026 2024-11-15T16:51:05,621 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:51:05,622 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/recovered.edits/0000000000000000026 2024-11-15T16:51:05,622 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e5381a957f04881a4eee5cd168aa7f50 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-15T16:51:05,637 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/.tmp/a/e9dd8396c8734b3891b389b8715afa5a is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731689465538/Put/seqid=0 2024-11-15T16:51:05,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741911_1089 (size=5523) 2024-11-15T16:51:05,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741911_1089 (size=5523) 2024-11-15T16:51:05,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741911_1089 (size=5523) 2024-11-15T16:51:05,644 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/.tmp/a/e9dd8396c8734b3891b389b8715afa5a 2024-11-15T16:51:05,666 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/.tmp/b/95f56fe982f94079972a3ba9defacf68 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731689465534/Put/seqid=0 2024-11-15T16:51:05,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741912_1090 (size=5524) 2024-11-15T16:51:05,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741912_1090 (size=5524) 2024-11-15T16:51:05,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741912_1090 (size=5524) 2024-11-15T16:51:05,673 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/.tmp/b/95f56fe982f94079972a3ba9defacf68 2024-11-15T16:51:05,692 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/.tmp/c/dc59e9759a4346568cc92a4b24b68739 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731689465536/Put/seqid=0 2024-11-15T16:51:05,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741913_1091 (size=5457) 2024-11-15T16:51:05,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741913_1091 (size=5457) 2024-11-15T16:51:05,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741913_1091 (size=5457) 2024-11-15T16:51:05,699 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/.tmp/c/dc59e9759a4346568cc92a4b24b68739 2024-11-15T16:51:05,704 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/.tmp/a/e9dd8396c8734b3891b389b8715afa5a as hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/a/e9dd8396c8734b3891b389b8715afa5a 2024-11-15T16:51:05,709 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/a/e9dd8396c8734b3891b389b8715afa5a, entries=7, sequenceid=26, filesize=5.4 K 2024-11-15T16:51:05,710 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/.tmp/b/95f56fe982f94079972a3ba9defacf68 as hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/b/95f56fe982f94079972a3ba9defacf68 2024-11-15T16:51:05,715 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/b/95f56fe982f94079972a3ba9defacf68, entries=7, sequenceid=26, filesize=5.4 K 2024-11-15T16:51:05,716 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/.tmp/c/dc59e9759a4346568cc92a4b24b68739 as hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/c/dc59e9759a4346568cc92a4b24b68739 2024-11-15T16:51:05,721 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/c/dc59e9759a4346568cc92a4b24b68739, entries=6, sequenceid=26, filesize=5.3 K 2024-11-15T16:51:05,721 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for e5381a957f04881a4eee5cd168aa7f50 in 99ms, sequenceid=26, compaction requested=false; wal=null 2024-11-15T16:51:05,722 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/recovered.edits/0000000000000000026 2024-11-15T16:51:05,723 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:51:05,723 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:51:05,723 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:51:05,725 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:51:05,727 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testReplayEditsAfterAbortingFlush/e5381a957f04881a4eee5cd168aa7f50/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-15T16:51:05,727 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e5381a957f04881a4eee5cd168aa7f50; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68082733, jitterRate=0.014511778950691223}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:51:05,728 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e5381a957f04881a4eee5cd168aa7f50 2024-11-15T16:51:05,728 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e5381a957f04881a4eee5cd168aa7f50: Running coprocessor pre-open hook at 1731689465609Writing region info on filesystem at 1731689465609Initializing all the Stores at 1731689465610 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689465610Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689465610Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689465610Obtaining lock to block concurrent updates at 1731689465622 (+12 ms)Preparing flush snapshotting stores in e5381a957f04881a4eee5cd168aa7f50 at 1731689465622Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731689465622Flushing stores of testReplayEditsAfterAbortingFlush,,1731689456369.e5381a957f04881a4eee5cd168aa7f50. at 1731689465622Flushing e5381a957f04881a4eee5cd168aa7f50/a: creating writer at 1731689465622Flushing e5381a957f04881a4eee5cd168aa7f50/a: appending metadata at 1731689465637 (+15 ms)Flushing e5381a957f04881a4eee5cd168aa7f50/a: closing flushed file at 1731689465637Flushing e5381a957f04881a4eee5cd168aa7f50/b: creating writer at 1731689465650 (+13 ms)Flushing e5381a957f04881a4eee5cd168aa7f50/b: appending metadata at 1731689465665 (+15 ms)Flushing e5381a957f04881a4eee5cd168aa7f50/b: closing flushed file at 1731689465665Flushing e5381a957f04881a4eee5cd168aa7f50/c: creating writer at 1731689465678 (+13 ms)Flushing e5381a957f04881a4eee5cd168aa7f50/c: appending metadata at 1731689465692 (+14 ms)Flushing e5381a957f04881a4eee5cd168aa7f50/c: closing flushed file at 1731689465692Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72dd63f4: reopening flushed file at 1731689465703 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@468b4a50: reopening flushed file at 1731689465709 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1842ac81: reopening flushed file at 1731689465715 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for e5381a957f04881a4eee5cd168aa7f50 in 99ms, sequenceid=26, compaction requested=false; wal=null at 1731689465721 (+6 ms)Cleaning up temporary data from old regions at 1731689465723 (+2 ms)Running coprocessor post-open hooks at 1731689465728 (+5 ms)Region opened successfully at 1731689465728 2024-11-15T16:51:05,748 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=419 (was 418) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1263208986-172.17.0.2-1731689433335:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1263208986-172.17.0.2-1731689433335:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:58794 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:45390 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1263208986-172.17.0.2-1731689433335:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:45394 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:48764 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:58786 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:48750 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1182 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=194 (was 230), ProcessCount=11 (was 11), AvailableMemoryMB=5046 (was 5061) 2024-11-15T16:51:05,749 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-15T16:51:05,761 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=419, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=194, ProcessCount=11, AvailableMemoryMB=5045 2024-11-15T16:51:05,761 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-15T16:51:05,776 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:51:05,777 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:51:05,778 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:51:05,781 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-39425635, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/hregion-39425635, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:51:05,794 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-39425635/hregion-39425635.1731689465781, exclude list is [], retry=0 2024-11-15T16:51:05,796 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:51:05,797 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:51:05,797 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:51:05,799 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-39425635/hregion-39425635.1731689465781 2024-11-15T16:51:05,799 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:37569:37569)] 2024-11-15T16:51:05,799 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 568419b6b4ce49a0313feb66b666b777, NAME => 'testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40387/hbase 2024-11-15T16:51:05,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741915_1093 (size=61) 2024-11-15T16:51:05,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741915_1093 (size=61) 2024-11-15T16:51:05,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741915_1093 (size=61) 2024-11-15T16:51:05,812 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:51:05,814 INFO [StoreOpener-568419b6b4ce49a0313feb66b666b777-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:05,815 INFO [StoreOpener-568419b6b4ce49a0313feb66b666b777-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 568419b6b4ce49a0313feb66b666b777 columnFamilyName a 2024-11-15T16:51:05,815 DEBUG [StoreOpener-568419b6b4ce49a0313feb66b666b777-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:05,816 INFO [StoreOpener-568419b6b4ce49a0313feb66b666b777-1 {}] regionserver.HStore(327): Store=568419b6b4ce49a0313feb66b666b777/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:05,816 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:05,817 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:05,817 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:05,817 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:05,817 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:05,819 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:05,821 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:51:05,821 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 568419b6b4ce49a0313feb66b666b777; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75494080, jitterRate=0.12494945526123047}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T16:51:05,822 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 568419b6b4ce49a0313feb66b666b777: Writing region info on filesystem at 1731689465813Initializing all the Stores at 1731689465813Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689465813Cleaning up temporary data from old regions at 1731689465817 (+4 ms)Region opened successfully at 1731689465822 (+5 ms) 2024-11-15T16:51:05,822 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 568419b6b4ce49a0313feb66b666b777, disabling compactions & flushes 2024-11-15T16:51:05,822 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777. 2024-11-15T16:51:05,822 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777. 2024-11-15T16:51:05,822 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777. after waiting 0 ms 2024-11-15T16:51:05,822 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777. 2024-11-15T16:51:05,822 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777. 2024-11-15T16:51:05,822 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 568419b6b4ce49a0313feb66b666b777: Waiting for close lock at 1731689465822Disabling compacts and flushes for region at 1731689465822Disabling writes for close at 1731689465822Writing region close event to WAL at 1731689465822Closed at 1731689465822 2024-11-15T16:51:05,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741914_1092 (size=95) 2024-11-15T16:51:05,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741914_1092 (size=95) 2024-11-15T16:51:05,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741914_1092 (size=95) 2024-11-15T16:51:05,827 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-15T16:51:05,827 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-39425635:(num 1731689465781) 2024-11-15T16:51:05,827 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:51:05,829 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:51:05,842 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689465829, exclude list is [], retry=0 2024-11-15T16:51:05,845 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:51:05,845 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:51:05,845 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:51:05,847 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689465829 2024-11-15T16:51:05,847 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:37569:37569)] 2024-11-15T16:51:05,847 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 568419b6b4ce49a0313feb66b666b777, NAME => 'testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:51:05,847 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:51:05,847 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:05,848 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:05,849 INFO [StoreOpener-568419b6b4ce49a0313feb66b666b777-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:05,850 INFO [StoreOpener-568419b6b4ce49a0313feb66b666b777-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 568419b6b4ce49a0313feb66b666b777 columnFamilyName a 2024-11-15T16:51:05,850 DEBUG [StoreOpener-568419b6b4ce49a0313feb66b666b777-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:05,850 INFO [StoreOpener-568419b6b4ce49a0313feb66b666b777-1 {}] regionserver.HStore(327): Store=568419b6b4ce49a0313feb66b666b777/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:05,850 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:05,851 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:05,852 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:05,852 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:05,852 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:05,854 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:05,855 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 568419b6b4ce49a0313feb66b666b777; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58870033, jitterRate=-0.12276814877986908}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T16:51:05,855 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 568419b6b4ce49a0313feb66b666b777: Writing region info on filesystem at 1731689465848Initializing all the Stores at 1731689465848Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689465848Cleaning up temporary data from old regions at 1731689465852 (+4 ms)Region opened successfully at 1731689465855 (+3 ms) 2024-11-15T16:51:05,863 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 568419b6b4ce49a0313feb66b666b777, disabling compactions & flushes 2024-11-15T16:51:05,863 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777. 2024-11-15T16:51:05,863 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777. 2024-11-15T16:51:05,863 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777. after waiting 0 ms 2024-11-15T16:51:05,863 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777. 2024-11-15T16:51:05,864 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777. 2024-11-15T16:51:05,864 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777. 2024-11-15T16:51:05,864 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 568419b6b4ce49a0313feb66b666b777: Waiting for close lock at 1731689465863Disabling compacts and flushes for region at 1731689465863Disabling writes for close at 1731689465863Writing region close event to WAL at 1731689465864 (+1 ms)Closed at 1731689465864 2024-11-15T16:51:05,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741916_1094 (size=1050) 2024-11-15T16:51:05,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741916_1094 (size=1050) 2024-11-15T16:51:05,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741916_1094 (size=1050) 2024-11-15T16:51:05,887 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40387/hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689465829, size=1.0 K (1050bytes) 2024-11-15T16:51:05,887 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40387/hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689465829 2024-11-15T16:51:05,887 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40387/hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689465829 after 0ms 2024-11-15T16:51:05,890 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689465829: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:51:05,890 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40387/hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689465829 took 3ms 2024-11-15T16:51:05,892 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40387/hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689465829 so closing down 2024-11-15T16:51:05,892 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T16:51:05,893 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731689465829.temp 2024-11-15T16:51:05,895 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/recovered.edits/0000000000000000003-wal.1731689465829.temp 2024-11-15T16:51:05,895 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T16:51:05,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741917_1095 (size=1050) 2024-11-15T16:51:05,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741917_1095 (size=1050) 2024-11-15T16:51:05,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741917_1095 (size=1050) 2024-11-15T16:51:05,903 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/recovered.edits/0000000000000000003-wal.1731689465829.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-15T16:51:05,905 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/recovered.edits/0000000000000000003-wal.1731689465829.temp to hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/recovered.edits/0000000000000000012 2024-11-15T16:51:05,905 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 14 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689465829, size=1.0 K, length=1050, corrupted=false, cancelled=false 2024-11-15T16:51:05,905 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40387/hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689465829, journal: Splitting hdfs://localhost:40387/hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689465829, size=1.0 K (1050bytes) at 1731689465887Finishing writing output for hdfs://localhost:40387/hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689465829 so closing down at 1731689465892 (+5 ms)Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/recovered.edits/0000000000000000003-wal.1731689465829.temp at 1731689465895 (+3 ms)3 split writer threads finished at 1731689465895Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/recovered.edits/0000000000000000003-wal.1731689465829.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731689465903 (+8 ms)Rename recovered edits hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/recovered.edits/0000000000000000003-wal.1731689465829.temp to hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/recovered.edits/0000000000000000012 at 1731689465905 (+2 ms)Processed 10 edits across 1 Regions in 14 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689465829, size=1.0 K, length=1050, corrupted=false, cancelled=false at 1731689465905 2024-11-15T16:51:05,907 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40387/hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689465829 to hdfs://localhost:40387/hbase/oldWALs/wal.1731689465829 2024-11-15T16:51:05,908 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/recovered.edits/0000000000000000012 2024-11-15T16:51:05,911 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:51:06,318 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:51:06,321 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:51:06,335 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689466321, exclude list is [], retry=0 2024-11-15T16:51:06,338 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:51:06,338 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:51:06,338 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:51:06,340 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689466321 2024-11-15T16:51:06,340 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:36277:36277)] 2024-11-15T16:51:06,340 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 568419b6b4ce49a0313feb66b666b777, NAME => 'testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:51:06,340 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:51:06,341 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,341 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,344 INFO [StoreOpener-568419b6b4ce49a0313feb66b666b777-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,345 INFO [StoreOpener-568419b6b4ce49a0313feb66b666b777-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 568419b6b4ce49a0313feb66b666b777 columnFamilyName a 2024-11-15T16:51:06,345 DEBUG [StoreOpener-568419b6b4ce49a0313feb66b666b777-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:06,346 INFO [StoreOpener-568419b6b4ce49a0313feb66b666b777-1 {}] regionserver.HStore(327): Store=568419b6b4ce49a0313feb66b666b777/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:06,347 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,347 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,348 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,349 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/recovered.edits/0000000000000000012 2024-11-15T16:51:06,351 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:51:06,352 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/recovered.edits/0000000000000000012 2024-11-15T16:51:06,352 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 568419b6b4ce49a0313feb66b666b777 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-15T16:51:06,370 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/.tmp/a/0d1f2ffd8a164cc1bd4cb28b5ca9e819 is 79, key is testDatalossWhenInputError/a:x0/1731689465855/Put/seqid=0 2024-11-15T16:51:06,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741919_1097 (size=5808) 2024-11-15T16:51:06,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741919_1097 (size=5808) 2024-11-15T16:51:06,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741919_1097 (size=5808) 2024-11-15T16:51:06,377 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/.tmp/a/0d1f2ffd8a164cc1bd4cb28b5ca9e819 2024-11-15T16:51:06,387 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/.tmp/a/0d1f2ffd8a164cc1bd4cb28b5ca9e819 as hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/a/0d1f2ffd8a164cc1bd4cb28b5ca9e819 2024-11-15T16:51:06,396 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/a/0d1f2ffd8a164cc1bd4cb28b5ca9e819, entries=10, sequenceid=12, filesize=5.7 K 2024-11-15T16:51:06,396 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 568419b6b4ce49a0313feb66b666b777 in 44ms, sequenceid=12, compaction requested=false; wal=null 2024-11-15T16:51:06,397 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/recovered.edits/0000000000000000012 2024-11-15T16:51:06,397 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,397 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,401 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,403 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-15T16:51:06,405 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 568419b6b4ce49a0313feb66b666b777; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72611643, jitterRate=0.08199779689311981}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T16:51:06,405 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 568419b6b4ce49a0313feb66b666b777: Writing region info on filesystem at 1731689466341Initializing all the Stores at 1731689466343 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689466343Obtaining lock to block concurrent updates at 1731689466352 (+9 ms)Preparing flush snapshotting stores in 568419b6b4ce49a0313feb66b666b777 at 1731689466352Finished memstore snapshotting testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731689466353 (+1 ms)Flushing stores of testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777. at 1731689466353Flushing 568419b6b4ce49a0313feb66b666b777/a: creating writer at 1731689466353Flushing 568419b6b4ce49a0313feb66b666b777/a: appending metadata at 1731689466369 (+16 ms)Flushing 568419b6b4ce49a0313feb66b666b777/a: closing flushed file at 1731689466369Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e543966: reopening flushed file at 1731689466385 (+16 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 568419b6b4ce49a0313feb66b666b777 in 44ms, sequenceid=12, compaction requested=false; wal=null at 1731689466396 (+11 ms)Cleaning up temporary data from old regions at 1731689466397 (+1 ms)Region opened successfully at 1731689466405 (+8 ms) 2024-11-15T16:51:06,408 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 568419b6b4ce49a0313feb66b666b777, NAME => 'testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:51:06,409 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731689465776.568419b6b4ce49a0313feb66b666b777.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:51:06,409 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,409 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,410 INFO [StoreOpener-568419b6b4ce49a0313feb66b666b777-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,411 INFO [StoreOpener-568419b6b4ce49a0313feb66b666b777-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 568419b6b4ce49a0313feb66b666b777 columnFamilyName a 2024-11-15T16:51:06,411 DEBUG [StoreOpener-568419b6b4ce49a0313feb66b666b777-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:06,416 DEBUG [StoreOpener-568419b6b4ce49a0313feb66b666b777-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/a/0d1f2ffd8a164cc1bd4cb28b5ca9e819 2024-11-15T16:51:06,416 INFO [StoreOpener-568419b6b4ce49a0313feb66b666b777-1 {}] regionserver.HStore(327): Store=568419b6b4ce49a0313feb66b666b777/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:06,416 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,417 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,418 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,419 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,419 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,420 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 568419b6b4ce49a0313feb66b666b777 2024-11-15T16:51:06,423 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testDatalossWhenInputError/568419b6b4ce49a0313feb66b666b777/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-15T16:51:06,424 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 568419b6b4ce49a0313feb66b666b777; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66282889, jitterRate=-0.012307986617088318}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T16:51:06,424 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 568419b6b4ce49a0313feb66b666b777: Writing region info on filesystem at 1731689466409Initializing all the Stores at 1731689466410 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689466410Cleaning up temporary data from old regions at 1731689466419 (+9 ms)Region opened successfully at 1731689466424 (+5 ms) 2024-11-15T16:51:06,444 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=429 (was 419) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:58880 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1263208986-172.17.0.2-1731689433335:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1263208986-172.17.0.2-1731689433335:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:45480 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1263208986-172.17.0.2-1731689433335:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:45394 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:48820 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:48764 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:58786 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=194 (was 194), ProcessCount=11 (was 11), AvailableMemoryMB=5027 (was 5045) 2024-11-15T16:51:06,445 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-15T16:51:06,460 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=429, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=194, ProcessCount=11, AvailableMemoryMB=5026 2024-11-15T16:51:06,460 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-15T16:51:06,475 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:51:06,477 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:51:06,477 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:51:06,480 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-85815141, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/hregion-85815141, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:51:06,493 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-85815141/hregion-85815141.1731689466480, exclude list is [], retry=0 2024-11-15T16:51:06,496 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:51:06,496 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:51:06,497 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:51:06,501 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-85815141/hregion-85815141.1731689466480 2024-11-15T16:51:06,502 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:37569:37569)] 2024-11-15T16:51:06,502 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 33c9c3b714ea5cadd8ad67582765c8ef, NAME => 'testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40387/hbase 2024-11-15T16:51:06,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741921_1099 (size=63) 2024-11-15T16:51:06,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741921_1099 (size=63) 2024-11-15T16:51:06,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741921_1099 (size=63) 2024-11-15T16:51:06,511 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:51:06,513 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,514 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33c9c3b714ea5cadd8ad67582765c8ef columnFamilyName a 2024-11-15T16:51:06,514 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:06,515 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(327): Store=33c9c3b714ea5cadd8ad67582765c8ef/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:06,515 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,516 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33c9c3b714ea5cadd8ad67582765c8ef columnFamilyName b 2024-11-15T16:51:06,516 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:06,516 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(327): Store=33c9c3b714ea5cadd8ad67582765c8ef/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:06,517 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,518 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33c9c3b714ea5cadd8ad67582765c8ef columnFamilyName c 2024-11-15T16:51:06,518 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:06,518 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(327): Store=33c9c3b714ea5cadd8ad67582765c8ef/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:06,518 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,519 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,519 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,520 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,520 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,521 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:51:06,522 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,524 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:51:06,524 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 33c9c3b714ea5cadd8ad67582765c8ef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61463143, jitterRate=-0.08412779867649078}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:51:06,525 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 33c9c3b714ea5cadd8ad67582765c8ef: Writing region info on filesystem at 1731689466511Initializing all the Stores at 1731689466512 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689466512Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689466512Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689466512Cleaning up temporary data from old regions at 1731689466520 (+8 ms)Region opened successfully at 1731689466525 (+5 ms) 2024-11-15T16:51:06,525 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 33c9c3b714ea5cadd8ad67582765c8ef, disabling compactions & flushes 2024-11-15T16:51:06,525 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef. 2024-11-15T16:51:06,525 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef. 2024-11-15T16:51:06,525 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef. after waiting 0 ms 2024-11-15T16:51:06,525 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef. 2024-11-15T16:51:06,525 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef. 2024-11-15T16:51:06,525 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 33c9c3b714ea5cadd8ad67582765c8ef: Waiting for close lock at 1731689466525Disabling compacts and flushes for region at 1731689466525Disabling writes for close at 1731689466525Writing region close event to WAL at 1731689466525Closed at 1731689466525 2024-11-15T16:51:06,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741920_1098 (size=95) 2024-11-15T16:51:06,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741920_1098 (size=95) 2024-11-15T16:51:06,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741920_1098 (size=95) 2024-11-15T16:51:06,531 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-15T16:51:06,531 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-85815141:(num 1731689466480) 2024-11-15T16:51:06,532 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:51:06,533 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:51:06,546 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533, exclude list is [], retry=0 2024-11-15T16:51:06,549 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:51:06,549 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:51:06,549 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:51:06,551 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533 2024-11-15T16:51:06,551 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:37569:37569)] 2024-11-15T16:51:06,551 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 33c9c3b714ea5cadd8ad67582765c8ef, NAME => 'testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:51:06,551 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:51:06,551 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,551 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,553 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,554 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33c9c3b714ea5cadd8ad67582765c8ef columnFamilyName a 2024-11-15T16:51:06,554 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:06,554 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(327): Store=33c9c3b714ea5cadd8ad67582765c8ef/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:06,554 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,555 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33c9c3b714ea5cadd8ad67582765c8ef columnFamilyName b 2024-11-15T16:51:06,555 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:06,556 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(327): Store=33c9c3b714ea5cadd8ad67582765c8ef/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:06,556 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,557 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33c9c3b714ea5cadd8ad67582765c8ef columnFamilyName c 2024-11-15T16:51:06,557 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:06,557 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(327): Store=33c9c3b714ea5cadd8ad67582765c8ef/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:06,558 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,558 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,559 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,560 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,560 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,561 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:51:06,562 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:06,563 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 33c9c3b714ea5cadd8ad67582765c8ef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59899576, jitterRate=-0.10742676258087158}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:51:06,564 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 33c9c3b714ea5cadd8ad67582765c8ef: Writing region info on filesystem at 1731689466552Initializing all the Stores at 1731689466552Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689466552Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689466553 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689466553Cleaning up temporary data from old regions at 1731689466560 (+7 ms)Region opened successfully at 1731689466564 (+4 ms) 2024-11-15T16:51:06,568 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731689466567/Put/seqid=0 2024-11-15T16:51:06,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741923_1101 (size=4875) 2024-11-15T16:51:06,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741923_1101 (size=4875) 2024-11-15T16:51:06,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741923_1101 (size=4875) 2024-11-15T16:51:06,577 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731689466577/Put/seqid=0 2024-11-15T16:51:06,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741924_1102 (size=4875) 2024-11-15T16:51:06,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741924_1102 (size=4875) 2024-11-15T16:51:06,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741924_1102 (size=4875) 2024-11-15T16:51:06,591 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731689466590/Put/seqid=0 2024-11-15T16:51:06,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741925_1103 (size=4875) 2024-11-15T16:51:06,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741925_1103 (size=4875) 2024-11-15T16:51:06,598 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40387/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 33c9c3b714ea5cadd8ad67582765c8ef/a 2024-11-15T16:51:06,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741925_1103 (size=4875) 2024-11-15T16:51:06,602 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-15T16:51:06,602 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-15T16:51:06,602 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40387/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 33c9c3b714ea5cadd8ad67582765c8ef/a 2024-11-15T16:51:06,606 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-15T16:51:06,606 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-15T16:51:06,606 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40387/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 33c9c3b714ea5cadd8ad67582765c8ef/a 2024-11-15T16:51:06,610 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-15T16:51:06,610 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-15T16:51:06,610 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 33c9c3b714ea5cadd8ad67582765c8ef 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-15T16:51:06,625 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/.tmp/a/4b413d0f50b94922b9496332079793b8 is 55, key is testCompactedBulkLoadedFiles/a:a/1731689466564/Put/seqid=0 2024-11-15T16:51:06,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741926_1104 (size=5107) 2024-11-15T16:51:06,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741926_1104 (size=5107) 2024-11-15T16:51:06,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741926_1104 (size=5107) 2024-11-15T16:51:06,636 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/.tmp/a/4b413d0f50b94922b9496332079793b8 2024-11-15T16:51:06,642 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/.tmp/a/4b413d0f50b94922b9496332079793b8 as hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/4b413d0f50b94922b9496332079793b8 2024-11-15T16:51:06,647 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/4b413d0f50b94922b9496332079793b8, entries=1, sequenceid=4, filesize=5.0 K 2024-11-15T16:51:06,648 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 33c9c3b714ea5cadd8ad67582765c8ef in 38ms, sequenceid=4, compaction requested=false 2024-11-15T16:51:06,648 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 33c9c3b714ea5cadd8ad67582765c8ef: 2024-11-15T16:51:06,650 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/f9315f0e98db41078fc0307deba6159d_SeqId_4_ 2024-11-15T16:51:06,651 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/a4fe6a8a056344448f1463ec90a6e209_SeqId_4_ 2024-11-15T16:51:06,652 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/114b9da6433e473191264346e357c706_SeqId_4_ 2024-11-15T16:51:06,652 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40387/hbase/testCompactedBulkLoadedFiles/hfile0 into 33c9c3b714ea5cadd8ad67582765c8ef/a as hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/f9315f0e98db41078fc0307deba6159d_SeqId_4_ - updating store file list. 2024-11-15T16:51:06,657 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for f9315f0e98db41078fc0307deba6159d_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-15T16:51:06,657 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/f9315f0e98db41078fc0307deba6159d_SeqId_4_ into 33c9c3b714ea5cadd8ad67582765c8ef/a 2024-11-15T16:51:06,657 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40387/hbase/testCompactedBulkLoadedFiles/hfile0 into 33c9c3b714ea5cadd8ad67582765c8ef/a (new location: hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/f9315f0e98db41078fc0307deba6159d_SeqId_4_) 2024-11-15T16:51:06,658 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40387/hbase/testCompactedBulkLoadedFiles/hfile1 into 33c9c3b714ea5cadd8ad67582765c8ef/a as hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/a4fe6a8a056344448f1463ec90a6e209_SeqId_4_ - updating store file list. 2024-11-15T16:51:06,662 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for a4fe6a8a056344448f1463ec90a6e209_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-15T16:51:06,663 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/a4fe6a8a056344448f1463ec90a6e209_SeqId_4_ into 33c9c3b714ea5cadd8ad67582765c8ef/a 2024-11-15T16:51:06,663 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40387/hbase/testCompactedBulkLoadedFiles/hfile1 into 33c9c3b714ea5cadd8ad67582765c8ef/a (new location: hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/a4fe6a8a056344448f1463ec90a6e209_SeqId_4_) 2024-11-15T16:51:06,663 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40387/hbase/testCompactedBulkLoadedFiles/hfile2 into 33c9c3b714ea5cadd8ad67582765c8ef/a as hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/114b9da6433e473191264346e357c706_SeqId_4_ - updating store file list. 2024-11-15T16:51:06,668 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 114b9da6433e473191264346e357c706_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-15T16:51:06,668 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/114b9da6433e473191264346e357c706_SeqId_4_ into 33c9c3b714ea5cadd8ad67582765c8ef/a 2024-11-15T16:51:06,668 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40387/hbase/testCompactedBulkLoadedFiles/hfile2 into 33c9c3b714ea5cadd8ad67582765c8ef/a (new location: hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/114b9da6433e473191264346e357c706_SeqId_4_) 2024-11-15T16:51:06,676 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:51:06,676 DEBUG [Time-limited test {}] regionserver.HStore(1541): 33c9c3b714ea5cadd8ad67582765c8ef/a is initiating major compaction (all files) 2024-11-15T16:51:06,676 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 33c9c3b714ea5cadd8ad67582765c8ef/a in testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef. 2024-11-15T16:51:06,676 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/4b413d0f50b94922b9496332079793b8, hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/f9315f0e98db41078fc0307deba6159d_SeqId_4_, hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/a4fe6a8a056344448f1463ec90a6e209_SeqId_4_, hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/114b9da6433e473191264346e357c706_SeqId_4_] into tmpdir=hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/.tmp, totalSize=19.3 K 2024-11-15T16:51:06,677 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4b413d0f50b94922b9496332079793b8, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731689466564 2024-11-15T16:51:06,677 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f9315f0e98db41078fc0307deba6159d_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-15T16:51:06,677 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a4fe6a8a056344448f1463ec90a6e209_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-15T16:51:06,678 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 114b9da6433e473191264346e357c706_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-15T16:51:06,690 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/.tmp/a/42f5a0cbafbc4e5fb5b6ac758f6e61b7 is 55, key is testCompactedBulkLoadedFiles/a:a/1731689466564/Put/seqid=0 2024-11-15T16:51:06,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741927_1105 (size=6154) 2024-11-15T16:51:06,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741927_1105 (size=6154) 2024-11-15T16:51:06,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741927_1105 (size=6154) 2024-11-15T16:51:06,703 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/.tmp/a/42f5a0cbafbc4e5fb5b6ac758f6e61b7 as hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/42f5a0cbafbc4e5fb5b6ac758f6e61b7 2024-11-15T16:51:06,709 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 33c9c3b714ea5cadd8ad67582765c8ef/a of 33c9c3b714ea5cadd8ad67582765c8ef into 42f5a0cbafbc4e5fb5b6ac758f6e61b7(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:51:06,709 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 33c9c3b714ea5cadd8ad67582765c8ef: 2024-11-15T16:51:06,710 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-15T16:51:06,710 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-15T16:51:06,742 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40387/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533, size=0 (0bytes) 2024-11-15T16:51:06,743 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40387/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533 might be still open, length is 0 2024-11-15T16:51:06,743 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40387/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533 2024-11-15T16:51:06,743 WARN [IPC Server handler 1 on default port 40387 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-15T16:51:06,743 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40387/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533 after 0ms 2024-11-15T16:51:06,862 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-15T16:51:06,862 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-15T16:51:06,863 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-15T16:51:06,863 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-15T16:51:08,423 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:45510 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:39325:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45510 dst: /127.0.0.1:39325 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39325 remote=/127.0.0.1:45510]. Total timeout mills is 60000, 58286 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:51:08,424 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:48846 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:42975:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48846 dst: /127.0.0.1:42975 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:51:08,424 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:58914 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:37733:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58914 dst: /127.0.0.1:37733 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:51:08,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741922_1106 (size=1175) 2024-11-15T16:51:08,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741922_1106 (size=1175) 2024-11-15T16:51:10,744 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40387/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533 after 4001ms 2024-11-15T16:51:10,747 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:51:10,747 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40387/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533 took 4005ms 2024-11-15T16:51:10,749 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40387/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533; continuing. 2024-11-15T16:51:10,749 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40387/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533 so closing down 2024-11-15T16:51:10,749 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T16:51:10,750 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731689466533.temp 2024-11-15T16:51:10,752 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/recovered.edits/0000000000000000003-wal.1731689466533.temp 2024-11-15T16:51:10,752 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T16:51:10,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741928_1107 (size=550) 2024-11-15T16:51:10,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741928_1107 (size=550) 2024-11-15T16:51:10,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741928_1107 (size=550) 2024-11-15T16:51:10,759 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/recovered.edits/0000000000000000003-wal.1731689466533.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-15T16:51:10,761 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/recovered.edits/0000000000000000003-wal.1731689466533.temp to hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/recovered.edits/0000000000000000008 2024-11-15T16:51:10,761 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:40387/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533, size=0, length=0, corrupted=false, cancelled=false 2024-11-15T16:51:10,761 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40387/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533, journal: Splitting hdfs://localhost:40387/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533, size=0 (0bytes) at 1731689466743Finishing writing output for hdfs://localhost:40387/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533 so closing down at 1731689470749 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/recovered.edits/0000000000000000003-wal.1731689466533.temp at 1731689470752 (+3 ms)3 split writer threads finished at 1731689470752Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/recovered.edits/0000000000000000003-wal.1731689466533.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731689470759 (+7 ms)Rename recovered edits hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/recovered.edits/0000000000000000003-wal.1731689466533.temp to hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/recovered.edits/0000000000000000008 at 1731689470761 (+2 ms)Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:40387/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533, size=0, length=0, corrupted=false, cancelled=false at 1731689470761 2024-11-15T16:51:10,762 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40387/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533 to hdfs://localhost:40387/hbase/oldWALs/wal.1731689466533 2024-11-15T16:51:10,763 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/recovered.edits/0000000000000000008 2024-11-15T16:51:10,763 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:51:10,765 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:51:10,780 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689470765, exclude list is [], retry=0 2024-11-15T16:51:10,782 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:51:10,783 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:51:10,783 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:51:10,784 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689470765 2024-11-15T16:51:10,784 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:36277:36277)] 2024-11-15T16:51:10,785 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 33c9c3b714ea5cadd8ad67582765c8ef, NAME => 'testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:51:10,785 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:51:10,785 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:10,785 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:10,786 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:10,787 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33c9c3b714ea5cadd8ad67582765c8ef columnFamilyName a 2024-11-15T16:51:10,787 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:10,793 DEBUG [StoreFileOpener-33c9c3b714ea5cadd8ad67582765c8ef-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 114b9da6433e473191264346e357c706_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-15T16:51:10,794 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/114b9da6433e473191264346e357c706_SeqId_4_ 2024-11-15T16:51:10,797 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/42f5a0cbafbc4e5fb5b6ac758f6e61b7 2024-11-15T16:51:10,800 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/4b413d0f50b94922b9496332079793b8 2024-11-15T16:51:10,803 DEBUG [StoreFileOpener-33c9c3b714ea5cadd8ad67582765c8ef-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for a4fe6a8a056344448f1463ec90a6e209_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-15T16:51:10,804 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/a4fe6a8a056344448f1463ec90a6e209_SeqId_4_ 2024-11-15T16:51:10,807 DEBUG [StoreFileOpener-33c9c3b714ea5cadd8ad67582765c8ef-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for f9315f0e98db41078fc0307deba6159d_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-15T16:51:10,807 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/f9315f0e98db41078fc0307deba6159d_SeqId_4_ 2024-11-15T16:51:10,807 WARN [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/114b9da6433e473191264346e357c706_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6fd9e869 2024-11-15T16:51:10,807 WARN [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/4b413d0f50b94922b9496332079793b8 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6fd9e869 2024-11-15T16:51:10,807 WARN [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/a4fe6a8a056344448f1463ec90a6e209_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6fd9e869 2024-11-15T16:51:10,807 WARN [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/f9315f0e98db41078fc0307deba6159d_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6fd9e869 2024-11-15T16:51:10,807 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/114b9da6433e473191264346e357c706_SeqId_4_, hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/4b413d0f50b94922b9496332079793b8, hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/a4fe6a8a056344448f1463ec90a6e209_SeqId_4_, hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/f9315f0e98db41078fc0307deba6159d_SeqId_4_] to archive 2024-11-15T16:51:10,808 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T16:51:10,809 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/114b9da6433e473191264346e357c706_SeqId_4_ to hdfs://localhost:40387/hbase/archive/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/114b9da6433e473191264346e357c706_SeqId_4_ 2024-11-15T16:51:10,810 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/4b413d0f50b94922b9496332079793b8 to hdfs://localhost:40387/hbase/archive/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/4b413d0f50b94922b9496332079793b8 2024-11-15T16:51:10,811 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/a4fe6a8a056344448f1463ec90a6e209_SeqId_4_ to hdfs://localhost:40387/hbase/archive/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/a4fe6a8a056344448f1463ec90a6e209_SeqId_4_ 2024-11-15T16:51:10,812 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/f9315f0e98db41078fc0307deba6159d_SeqId_4_ to hdfs://localhost:40387/hbase/archive/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/a/f9315f0e98db41078fc0307deba6159d_SeqId_4_ 2024-11-15T16:51:10,812 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(327): Store=33c9c3b714ea5cadd8ad67582765c8ef/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:10,812 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:10,813 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33c9c3b714ea5cadd8ad67582765c8ef columnFamilyName b 2024-11-15T16:51:10,813 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:10,814 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(327): Store=33c9c3b714ea5cadd8ad67582765c8ef/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:10,814 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:10,815 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33c9c3b714ea5cadd8ad67582765c8ef columnFamilyName c 2024-11-15T16:51:10,815 DEBUG [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:10,815 INFO [StoreOpener-33c9c3b714ea5cadd8ad67582765c8ef-1 {}] regionserver.HStore(327): Store=33c9c3b714ea5cadd8ad67582765c8ef/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:10,815 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:10,816 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:10,817 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:10,817 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/recovered.edits/0000000000000000008 2024-11-15T16:51:10,819 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:51:10,820 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 33c9c3b714ea5cadd8ad67582765c8ef : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "33c9c3b714ea5cadd8ad67582765c8ef" family_name: "a" compaction_input: "4b413d0f50b94922b9496332079793b8" compaction_input: "f9315f0e98db41078fc0307deba6159d_SeqId_4_" compaction_input: "a4fe6a8a056344448f1463ec90a6e209_SeqId_4_" compaction_input: "114b9da6433e473191264346e357c706_SeqId_4_" compaction_output: "42f5a0cbafbc4e5fb5b6ac758f6e61b7" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-15T16:51:10,820 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-15T16:51:10,820 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/recovered.edits/0000000000000000008 2024-11-15T16:51:10,821 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/recovered.edits/0000000000000000008 2024-11-15T16:51:10,822 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:10,822 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:10,822 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:51:10,824 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 33c9c3b714ea5cadd8ad67582765c8ef 2024-11-15T16:51:10,826 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testCompactedBulkLoadedFiles/33c9c3b714ea5cadd8ad67582765c8ef/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T16:51:10,827 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 33c9c3b714ea5cadd8ad67582765c8ef; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66658935, jitterRate=-0.006704464554786682}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:51:10,827 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 33c9c3b714ea5cadd8ad67582765c8ef: Writing region info on filesystem at 1731689470785Initializing all the Stores at 1731689470786 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689470786Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689470786Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689470786Cleaning up temporary data from old regions at 1731689470822 (+36 ms)Region opened successfully at 1731689470827 (+5 ms) 2024-11-15T16:51:10,829 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 33c9c3b714ea5cadd8ad67582765c8ef, disabling compactions & flushes 2024-11-15T16:51:10,829 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef. 2024-11-15T16:51:10,829 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef. 2024-11-15T16:51:10,829 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef. after waiting 0 ms 2024-11-15T16:51:10,829 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef. 2024-11-15T16:51:10,830 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731689466475.33c9c3b714ea5cadd8ad67582765c8ef. 2024-11-15T16:51:10,830 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 33c9c3b714ea5cadd8ad67582765c8ef: Waiting for close lock at 1731689470829Disabling compacts and flushes for region at 1731689470829Disabling writes for close at 1731689470829Writing region close event to WAL at 1731689470830 (+1 ms)Closed at 1731689470830 2024-11-15T16:51:10,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741929_1108 (size=95) 2024-11-15T16:51:10,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741929_1108 (size=95) 2024-11-15T16:51:10,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741929_1108 (size=95) 2024-11-15T16:51:10,836 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-15T16:51:10,836 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731689470765) 2024-11-15T16:51:10,852 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=442 (was 429) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1853009437_22 at /127.0.0.1:34222 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:40387 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1853009437_22 at /127.0.0.1:54860 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1396088596) connection to localhost/127.0.0.1:40387 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=179 (was 194), ProcessCount=11 (was 11), AvailableMemoryMB=5008 (was 5026) 2024-11-15T16:51:10,852 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-15T16:51:10,865 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=442, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=179, ProcessCount=11, AvailableMemoryMB=5008 2024-11-15T16:51:10,865 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-15T16:51:10,879 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:51:10,881 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:51:10,882 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:51:10,884 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-74201782, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/hregion-74201782, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:51:10,898 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-74201782/hregion-74201782.1731689470884, exclude list is [], retry=0 2024-11-15T16:51:10,900 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:51:10,901 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:51:10,901 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:51:10,931 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-74201782/hregion-74201782.1731689470884 2024-11-15T16:51:10,932 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:37569:37569),(127.0.0.1/127.0.0.1:36277:36277)] 2024-11-15T16:51:10,932 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8906c10313c834eb361983d788872810, NAME => 'testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40387/hbase 2024-11-15T16:51:10,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741931_1110 (size=67) 2024-11-15T16:51:10,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741931_1110 (size=67) 2024-11-15T16:51:10,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741931_1110 (size=67) 2024-11-15T16:51:10,946 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:51:10,947 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8906c10313c834eb361983d788872810 2024-11-15T16:51:10,948 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8906c10313c834eb361983d788872810 columnFamilyName a 2024-11-15T16:51:10,948 DEBUG [StoreOpener-8906c10313c834eb361983d788872810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:10,949 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(327): Store=8906c10313c834eb361983d788872810/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:10,949 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8906c10313c834eb361983d788872810 2024-11-15T16:51:10,950 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8906c10313c834eb361983d788872810 columnFamilyName b 2024-11-15T16:51:10,950 DEBUG [StoreOpener-8906c10313c834eb361983d788872810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:10,951 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(327): Store=8906c10313c834eb361983d788872810/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:10,951 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8906c10313c834eb361983d788872810 2024-11-15T16:51:10,952 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8906c10313c834eb361983d788872810 columnFamilyName c 2024-11-15T16:51:10,952 DEBUG [StoreOpener-8906c10313c834eb361983d788872810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:10,952 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(327): Store=8906c10313c834eb361983d788872810/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:10,952 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8906c10313c834eb361983d788872810 2024-11-15T16:51:10,953 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810 2024-11-15T16:51:10,953 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810 2024-11-15T16:51:10,954 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8906c10313c834eb361983d788872810 2024-11-15T16:51:10,954 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8906c10313c834eb361983d788872810 2024-11-15T16:51:10,954 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:51:10,955 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8906c10313c834eb361983d788872810 2024-11-15T16:51:10,957 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:51:10,958 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8906c10313c834eb361983d788872810; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71443244, jitterRate=0.0645872950553894}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:51:10,958 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8906c10313c834eb361983d788872810: Writing region info on filesystem at 1731689470946Initializing all the Stores at 1731689470947 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689470947Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689470947Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689470947Cleaning up temporary data from old regions at 1731689470954 (+7 ms)Region opened successfully at 1731689470958 (+4 ms) 2024-11-15T16:51:10,958 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8906c10313c834eb361983d788872810, disabling compactions & flushes 2024-11-15T16:51:10,958 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. 2024-11-15T16:51:10,958 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. 2024-11-15T16:51:10,958 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. after waiting 0 ms 2024-11-15T16:51:10,958 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. 2024-11-15T16:51:10,958 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. 2024-11-15T16:51:10,959 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8906c10313c834eb361983d788872810: Waiting for close lock at 1731689470958Disabling compacts and flushes for region at 1731689470958Disabling writes for close at 1731689470958Writing region close event to WAL at 1731689470958Closed at 1731689470958 2024-11-15T16:51:10,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741930_1109 (size=95) 2024-11-15T16:51:10,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741930_1109 (size=95) 2024-11-15T16:51:10,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741930_1109 (size=95) 2024-11-15T16:51:10,963 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-15T16:51:10,963 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-74201782:(num 1731689470884) 2024-11-15T16:51:10,964 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:51:10,965 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:51:10,980 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689470966, exclude list is [], retry=0 2024-11-15T16:51:10,983 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:51:10,983 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:51:10,983 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:51:10,985 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689470966 2024-11-15T16:51:10,985 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:37569:37569)] 2024-11-15T16:51:10,985 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8906c10313c834eb361983d788872810, NAME => 'testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:51:10,985 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:51:10,985 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8906c10313c834eb361983d788872810 2024-11-15T16:51:10,985 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8906c10313c834eb361983d788872810 2024-11-15T16:51:10,986 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8906c10313c834eb361983d788872810 2024-11-15T16:51:10,987 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8906c10313c834eb361983d788872810 columnFamilyName a 2024-11-15T16:51:10,987 DEBUG [StoreOpener-8906c10313c834eb361983d788872810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:10,988 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(327): Store=8906c10313c834eb361983d788872810/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:10,988 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8906c10313c834eb361983d788872810 2024-11-15T16:51:10,989 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8906c10313c834eb361983d788872810 columnFamilyName b 2024-11-15T16:51:10,989 DEBUG [StoreOpener-8906c10313c834eb361983d788872810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:10,989 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(327): Store=8906c10313c834eb361983d788872810/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:10,989 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8906c10313c834eb361983d788872810 2024-11-15T16:51:10,990 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8906c10313c834eb361983d788872810 columnFamilyName c 2024-11-15T16:51:10,990 DEBUG [StoreOpener-8906c10313c834eb361983d788872810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:10,990 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(327): Store=8906c10313c834eb361983d788872810/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:10,990 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8906c10313c834eb361983d788872810 2024-11-15T16:51:10,991 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810 2024-11-15T16:51:10,992 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810 2024-11-15T16:51:10,993 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8906c10313c834eb361983d788872810 2024-11-15T16:51:10,993 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8906c10313c834eb361983d788872810 2024-11-15T16:51:10,993 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:51:10,994 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8906c10313c834eb361983d788872810 2024-11-15T16:51:10,995 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8906c10313c834eb361983d788872810; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73489553, jitterRate=0.09507967531681061}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:51:10,996 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8906c10313c834eb361983d788872810: Writing region info on filesystem at 1731689470985Initializing all the Stores at 1731689470986 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689470986Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689470986Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689470986Cleaning up temporary data from old regions at 1731689470993 (+7 ms)Region opened successfully at 1731689470995 (+2 ms) 2024-11-15T16:51:11,004 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8906c10313c834eb361983d788872810 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-15T16:51:11,020 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/a/119ecf31190e414eb7aeb1d40e3ec3ba is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731689470996/Put/seqid=0 2024-11-15T16:51:11,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741933_1112 (size=5958) 2024-11-15T16:51:11,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741933_1112 (size=5958) 2024-11-15T16:51:11,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741933_1112 (size=5958) 2024-11-15T16:51:11,028 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/a/119ecf31190e414eb7aeb1d40e3ec3ba 2024-11-15T16:51:11,034 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/a/119ecf31190e414eb7aeb1d40e3ec3ba as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/a/119ecf31190e414eb7aeb1d40e3ec3ba 2024-11-15T16:51:11,039 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/a/119ecf31190e414eb7aeb1d40e3ec3ba, entries=10, sequenceid=13, filesize=5.8 K 2024-11-15T16:51:11,040 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 8906c10313c834eb361983d788872810 in 36ms, sequenceid=13, compaction requested=false 2024-11-15T16:51:11,040 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8906c10313c834eb361983d788872810: 2024-11-15T16:51:11,056 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8906c10313c834eb361983d788872810, disabling compactions & flushes 2024-11-15T16:51:11,056 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. 2024-11-15T16:51:11,056 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. 2024-11-15T16:51:11,056 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. after waiting 0 ms 2024-11-15T16:51:11,056 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. 2024-11-15T16:51:11,056 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. 2024-11-15T16:51:11,056 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. 2024-11-15T16:51:11,056 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8906c10313c834eb361983d788872810: Waiting for close lock at 1731689471056Disabling compacts and flushes for region at 1731689471056Disabling writes for close at 1731689471056Writing region close event to WAL at 1731689471056Closed at 1731689471056 2024-11-15T16:51:11,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741932_1111 (size=3343) 2024-11-15T16:51:11,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741932_1111 (size=3343) 2024-11-15T16:51:11,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741932_1111 (size=3343) 2024-11-15T16:51:11,072 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T16:51:11,082 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689470966, size=3.3 K (3343bytes) 2024-11-15T16:51:11,083 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689470966 2024-11-15T16:51:11,083 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689470966 after 0ms 2024-11-15T16:51:11,085 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689470966: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:51:11,085 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689470966 took 3ms 2024-11-15T16:51:11,087 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689470966 so closing down 2024-11-15T16:51:11,087 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T16:51:11,089 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731689470966.temp 2024-11-15T16:51:11,090 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000003-wal.1731689470966.temp 2024-11-15T16:51:11,090 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T16:51:11,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741934_1113 (size=2944) 2024-11-15T16:51:11,103 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000003-wal.1731689470966.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-15T16:51:11,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741934_1113 (size=2944) 2024-11-15T16:51:11,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741934_1113 (size=2944) 2024-11-15T16:51:11,106 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000003-wal.1731689470966.temp to hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000035 2024-11-15T16:51:11,106 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689470966, size=3.3 K, length=3343, corrupted=false, cancelled=false 2024-11-15T16:51:11,106 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689470966, journal: Splitting hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689470966, size=3.3 K (3343bytes) at 1731689471083Finishing writing output for hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689470966 so closing down at 1731689471087 (+4 ms)Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000003-wal.1731689470966.temp at 1731689471090 (+3 ms)3 split writer threads finished at 1731689471090Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000003-wal.1731689470966.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731689471103 (+13 ms)Rename recovered edits hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000003-wal.1731689470966.temp to hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000035 at 1731689471106 (+3 ms)Processed 32 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689470966, size=3.3 K, length=3343, corrupted=false, cancelled=false at 1731689471106 2024-11-15T16:51:11,108 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689470966 to hdfs://localhost:40387/hbase/oldWALs/wal.1731689470966 2024-11-15T16:51:11,109 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000035 2024-11-15T16:51:11,109 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:51:11,110 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:51:11,123 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111, exclude list is [], retry=0 2024-11-15T16:51:11,126 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:51:11,126 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:51:11,126 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:51:11,127 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111 2024-11-15T16:51:11,128 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:37569:37569)] 2024-11-15T16:51:11,128 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8906c10313c834eb361983d788872810, NAME => 'testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:51:11,128 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:51:11,128 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8906c10313c834eb361983d788872810 2024-11-15T16:51:11,128 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8906c10313c834eb361983d788872810 2024-11-15T16:51:11,129 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8906c10313c834eb361983d788872810 2024-11-15T16:51:11,130 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8906c10313c834eb361983d788872810 columnFamilyName a 2024-11-15T16:51:11,130 DEBUG [StoreOpener-8906c10313c834eb361983d788872810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:11,136 DEBUG [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/a/119ecf31190e414eb7aeb1d40e3ec3ba 2024-11-15T16:51:11,136 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(327): Store=8906c10313c834eb361983d788872810/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:11,136 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8906c10313c834eb361983d788872810 2024-11-15T16:51:11,137 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8906c10313c834eb361983d788872810 columnFamilyName b 2024-11-15T16:51:11,137 DEBUG [StoreOpener-8906c10313c834eb361983d788872810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:11,137 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(327): Store=8906c10313c834eb361983d788872810/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:11,138 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8906c10313c834eb361983d788872810 2024-11-15T16:51:11,138 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8906c10313c834eb361983d788872810 columnFamilyName c 2024-11-15T16:51:11,138 DEBUG [StoreOpener-8906c10313c834eb361983d788872810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:11,139 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(327): Store=8906c10313c834eb361983d788872810/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:11,139 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8906c10313c834eb361983d788872810 2024-11-15T16:51:11,139 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810 2024-11-15T16:51:11,141 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810 2024-11-15T16:51:11,141 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000035 2024-11-15T16:51:11,143 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:51:11,144 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000035 2024-11-15T16:51:11,144 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8906c10313c834eb361983d788872810 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-15T16:51:11,159 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/b/d9b481d98296457781c608e917b47912 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731689471040/Put/seqid=0 2024-11-15T16:51:11,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741936_1115 (size=5958) 2024-11-15T16:51:11,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741936_1115 (size=5958) 2024-11-15T16:51:11,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741936_1115 (size=5958) 2024-11-15T16:51:11,166 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/b/d9b481d98296457781c608e917b47912 2024-11-15T16:51:11,185 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/c/2266fc03bd3c4f1ea3b685d6ec5a56ca is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731689471046/Put/seqid=0 2024-11-15T16:51:11,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741937_1116 (size=5958) 2024-11-15T16:51:11,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741937_1116 (size=5958) 2024-11-15T16:51:11,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741937_1116 (size=5958) 2024-11-15T16:51:11,192 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/c/2266fc03bd3c4f1ea3b685d6ec5a56ca 2024-11-15T16:51:11,198 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/b/d9b481d98296457781c608e917b47912 as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/b/d9b481d98296457781c608e917b47912 2024-11-15T16:51:11,204 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/b/d9b481d98296457781c608e917b47912, entries=10, sequenceid=35, filesize=5.8 K 2024-11-15T16:51:11,205 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/c/2266fc03bd3c4f1ea3b685d6ec5a56ca as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/c/2266fc03bd3c4f1ea3b685d6ec5a56ca 2024-11-15T16:51:11,209 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/c/2266fc03bd3c4f1ea3b685d6ec5a56ca, entries=10, sequenceid=35, filesize=5.8 K 2024-11-15T16:51:11,210 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 8906c10313c834eb361983d788872810 in 66ms, sequenceid=35, compaction requested=false; wal=null 2024-11-15T16:51:11,210 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000035 2024-11-15T16:51:11,211 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8906c10313c834eb361983d788872810 2024-11-15T16:51:11,211 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8906c10313c834eb361983d788872810 2024-11-15T16:51:11,212 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:51:11,213 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8906c10313c834eb361983d788872810 2024-11-15T16:51:11,215 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-15T16:51:11,216 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8906c10313c834eb361983d788872810; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72396220, jitterRate=0.07878774404525757}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:51:11,217 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8906c10313c834eb361983d788872810: Writing region info on filesystem at 1731689471128Initializing all the Stores at 1731689471129 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689471129Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689471129Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689471129Obtaining lock to block concurrent updates at 1731689471144 (+15 ms)Preparing flush snapshotting stores in 8906c10313c834eb361983d788872810 at 1731689471144Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731689471145 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. at 1731689471145Flushing 8906c10313c834eb361983d788872810/b: creating writer at 1731689471145Flushing 8906c10313c834eb361983d788872810/b: appending metadata at 1731689471158 (+13 ms)Flushing 8906c10313c834eb361983d788872810/b: closing flushed file at 1731689471158Flushing 8906c10313c834eb361983d788872810/c: creating writer at 1731689471171 (+13 ms)Flushing 8906c10313c834eb361983d788872810/c: appending metadata at 1731689471185 (+14 ms)Flushing 8906c10313c834eb361983d788872810/c: closing flushed file at 1731689471185Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37c66419: reopening flushed file at 1731689471197 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b3a55ca: reopening flushed file at 1731689471204 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 8906c10313c834eb361983d788872810 in 66ms, sequenceid=35, compaction requested=false; wal=null at 1731689471210 (+6 ms)Cleaning up temporary data from old regions at 1731689471211 (+1 ms)Region opened successfully at 1731689471217 (+6 ms) 2024-11-15T16:51:11,286 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111, size=0 (0bytes) 2024-11-15T16:51:11,286 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111 might be still open, length is 0 2024-11-15T16:51:11,286 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111 2024-11-15T16:51:11,286 WARN [IPC Server handler 4 on default port 40387 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-15T16:51:11,287 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111 after 1ms 2024-11-15T16:51:11,422 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:33722 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:37733:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33722 dst: /127.0.0.1:37733 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37733 remote=/127.0.0.1:33722]. Total timeout mills is 60000, 59827 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:51:11,422 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:34292 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:42975:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34292 dst: /127.0.0.1:42975 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:51:11,423 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-390475633_22 at /127.0.0.1:54936 [Receiving block BP-1263208986-172.17.0.2-1731689433335:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:39325:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54936 dst: /127.0.0.1:39325 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:51:11,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741935_1117 (size=2936) 2024-11-15T16:51:11,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741935_1117 (size=2936) 2024-11-15T16:51:15,287 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111 after 4001ms 2024-11-15T16:51:15,290 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:51:15,290 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111 took 4004ms 2024-11-15T16:51:15,293 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111; continuing. 2024-11-15T16:51:15,293 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111 so closing down 2024-11-15T16:51:15,293 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T16:51:15,294 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731689471111.temp 2024-11-15T16:51:15,295 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000037-wal.1731689471111.temp 2024-11-15T16:51:15,295 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T16:51:15,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741938_1118 (size=2944) 2024-11-15T16:51:15,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741938_1118 (size=2944) 2024-11-15T16:51:15,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741938_1118 (size=2944) 2024-11-15T16:51:15,303 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000037-wal.1731689471111.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-15T16:51:15,304 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000037-wal.1731689471111.temp to hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000066 2024-11-15T16:51:15,304 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111, size=0, length=0, corrupted=false, cancelled=false 2024-11-15T16:51:15,305 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111, journal: Splitting hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111, size=0 (0bytes) at 1731689471286Finishing writing output for hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111 so closing down at 1731689475293 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000037-wal.1731689471111.temp at 1731689475295 (+2 ms)3 split writer threads finished at 1731689475295Closed recovered edits writer path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000037-wal.1731689471111.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731689475303 (+8 ms)Rename recovered edits hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000037-wal.1731689471111.temp to hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000066 at 1731689475304 (+1 ms)Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111, size=0, length=0, corrupted=false, cancelled=false at 1731689475305 (+1 ms) 2024-11-15T16:51:15,306 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111 to hdfs://localhost:40387/hbase/oldWALs/wal.1731689471111 2024-11-15T16:51:15,307 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000066 2024-11-15T16:51:15,307 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-15T16:51:15,309 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40387/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879, archiveDir=hdfs://localhost:40387/hbase/oldWALs, maxLogs=32 2024-11-15T16:51:15,322 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689475309, exclude list is [], retry=0 2024-11-15T16:51:15,324 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-8100e98b-4a9e-44f7-9998-a14437efca4b,DISK] 2024-11-15T16:51:15,325 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39325,DS-16a36ab3-4f1f-424d-bd7b-9ddf1e19dca3,DISK] 2024-11-15T16:51:15,325 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42975,DS-4629d782-bcf0-4860-8f68-d4ca2009a650,DISK] 2024-11-15T16:51:15,326 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689475309 2024-11-15T16:51:15,327 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36277:36277),(127.0.0.1/127.0.0.1:40741:40741),(127.0.0.1/127.0.0.1:37569:37569)] 2024-11-15T16:51:15,327 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:51:15,328 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8906c10313c834eb361983d788872810 2024-11-15T16:51:15,329 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8906c10313c834eb361983d788872810 columnFamilyName a 2024-11-15T16:51:15,329 DEBUG [StoreOpener-8906c10313c834eb361983d788872810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:15,335 DEBUG [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/a/119ecf31190e414eb7aeb1d40e3ec3ba 2024-11-15T16:51:15,335 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(327): Store=8906c10313c834eb361983d788872810/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:15,335 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8906c10313c834eb361983d788872810 2024-11-15T16:51:15,336 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8906c10313c834eb361983d788872810 columnFamilyName b 2024-11-15T16:51:15,336 DEBUG [StoreOpener-8906c10313c834eb361983d788872810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:15,340 DEBUG [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/b/d9b481d98296457781c608e917b47912 2024-11-15T16:51:15,341 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(327): Store=8906c10313c834eb361983d788872810/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:15,341 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8906c10313c834eb361983d788872810 2024-11-15T16:51:15,341 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8906c10313c834eb361983d788872810 columnFamilyName c 2024-11-15T16:51:15,341 DEBUG [StoreOpener-8906c10313c834eb361983d788872810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:51:15,345 DEBUG [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/c/2266fc03bd3c4f1ea3b685d6ec5a56ca 2024-11-15T16:51:15,345 INFO [StoreOpener-8906c10313c834eb361983d788872810-1 {}] regionserver.HStore(327): Store=8906c10313c834eb361983d788872810/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:51:15,346 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8906c10313c834eb361983d788872810 2024-11-15T16:51:15,346 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810 2024-11-15T16:51:15,348 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810 2024-11-15T16:51:15,348 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000066 2024-11-15T16:51:15,350 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-15T16:51:15,355 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000066 2024-11-15T16:51:15,355 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8906c10313c834eb361983d788872810 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-15T16:51:15,370 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/a/2ffed121d6804228b0b14045ce743eba is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731689471224/Put/seqid=0 2024-11-15T16:51:15,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741940_1120 (size=5958) 2024-11-15T16:51:15,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741940_1120 (size=5958) 2024-11-15T16:51:15,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741940_1120 (size=5958) 2024-11-15T16:51:15,377 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/a/2ffed121d6804228b0b14045ce743eba 2024-11-15T16:51:15,396 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/b/9bf9a0a77d8a431ba25492ad379c287f is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731689471233/Put/seqid=0 2024-11-15T16:51:15,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741941_1121 (size=5958) 2024-11-15T16:51:15,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741941_1121 (size=5958) 2024-11-15T16:51:15,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741941_1121 (size=5958) 2024-11-15T16:51:15,403 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/b/9bf9a0a77d8a431ba25492ad379c287f 2024-11-15T16:51:15,422 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/c/c81f638ac92d498fafb648093d1162e0 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731689471241/Put/seqid=0 2024-11-15T16:51:15,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741942_1122 (size=5958) 2024-11-15T16:51:15,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741942_1122 (size=5958) 2024-11-15T16:51:15,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741942_1122 (size=5958) 2024-11-15T16:51:15,429 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/c/c81f638ac92d498fafb648093d1162e0 2024-11-15T16:51:15,434 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/a/2ffed121d6804228b0b14045ce743eba as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/a/2ffed121d6804228b0b14045ce743eba 2024-11-15T16:51:15,438 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/a/2ffed121d6804228b0b14045ce743eba, entries=10, sequenceid=66, filesize=5.8 K 2024-11-15T16:51:15,439 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/b/9bf9a0a77d8a431ba25492ad379c287f as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/b/9bf9a0a77d8a431ba25492ad379c287f 2024-11-15T16:51:15,443 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/b/9bf9a0a77d8a431ba25492ad379c287f, entries=10, sequenceid=66, filesize=5.8 K 2024-11-15T16:51:15,444 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/.tmp/c/c81f638ac92d498fafb648093d1162e0 as hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/c/c81f638ac92d498fafb648093d1162e0 2024-11-15T16:51:15,448 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/c/c81f638ac92d498fafb648093d1162e0, entries=10, sequenceid=66, filesize=5.8 K 2024-11-15T16:51:15,448 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 8906c10313c834eb361983d788872810 in 93ms, sequenceid=66, compaction requested=false; wal=null 2024-11-15T16:51:15,449 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/0000000000000000066 2024-11-15T16:51:15,449 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8906c10313c834eb361983d788872810 2024-11-15T16:51:15,449 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8906c10313c834eb361983d788872810 2024-11-15T16:51:15,450 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T16:51:15,451 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8906c10313c834eb361983d788872810 2024-11-15T16:51:15,453 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/hbase/data/default/testReplayEditsWrittenViaHRegion/8906c10313c834eb361983d788872810/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-15T16:51:15,454 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8906c10313c834eb361983d788872810; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60861819, jitterRate=-0.09308822453022003}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T16:51:15,454 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8906c10313c834eb361983d788872810: Writing region info on filesystem at 1731689475327Initializing all the Stores at 1731689475328 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689475328Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689475328Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689475328Obtaining lock to block concurrent updates at 1731689475355 (+27 ms)Preparing flush snapshotting stores in 8906c10313c834eb361983d788872810 at 1731689475355Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731689475355Flushing stores of testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. at 1731689475355Flushing 8906c10313c834eb361983d788872810/a: creating writer at 1731689475355Flushing 8906c10313c834eb361983d788872810/a: appending metadata at 1731689475370 (+15 ms)Flushing 8906c10313c834eb361983d788872810/a: closing flushed file at 1731689475370Flushing 8906c10313c834eb361983d788872810/b: creating writer at 1731689475381 (+11 ms)Flushing 8906c10313c834eb361983d788872810/b: appending metadata at 1731689475395 (+14 ms)Flushing 8906c10313c834eb361983d788872810/b: closing flushed file at 1731689475395Flushing 8906c10313c834eb361983d788872810/c: creating writer at 1731689475407 (+12 ms)Flushing 8906c10313c834eb361983d788872810/c: appending metadata at 1731689475421 (+14 ms)Flushing 8906c10313c834eb361983d788872810/c: closing flushed file at 1731689475421Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26c822f: reopening flushed file at 1731689475433 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@123df95a: reopening flushed file at 1731689475438 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cb145f: reopening flushed file at 1731689475443 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 8906c10313c834eb361983d788872810 in 93ms, sequenceid=66, compaction requested=false; wal=null at 1731689475448 (+5 ms)Cleaning up temporary data from old regions at 1731689475449 (+1 ms)Region opened successfully at 1731689475454 (+5 ms) 2024-11-15T16:51:15,465 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8906c10313c834eb361983d788872810, disabling compactions & flushes 2024-11-15T16:51:15,465 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. 2024-11-15T16:51:15,465 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. 2024-11-15T16:51:15,466 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. after waiting 0 ms 2024-11-15T16:51:15,466 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. 2024-11-15T16:51:15,467 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731689470880.8906c10313c834eb361983d788872810. 2024-11-15T16:51:15,467 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8906c10313c834eb361983d788872810: Waiting for close lock at 1731689475465Disabling compacts and flushes for region at 1731689475465Disabling writes for close at 1731689475466 (+1 ms)Writing region close event to WAL at 1731689475467 (+1 ms)Closed at 1731689475467 2024-11-15T16:51:15,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741939_1119 (size=95) 2024-11-15T16:51:15,470 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689475309 not finished, retry = 0 2024-11-15T16:51:15,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741939_1119 (size=95) 2024-11-15T16:51:15,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741939_1119 (size=95) 2024-11-15T16:51:15,573 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-15T16:51:15,573 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731689475309) 2024-11-15T16:51:15,588 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=447 (was 442) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:40387 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1468362615_22 at /127.0.0.1:54984 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1396088596) connection to localhost/127.0.0.1:40387 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1468362615_22 at /127.0.0.1:34324 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1468362615_22 at /127.0.0.1:33744 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=164 (was 179), ProcessCount=11 (was 11), AvailableMemoryMB=4969 (was 5008) 2024-11-15T16:51:15,588 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-15T16:51:15,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T16:51:15,588 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T16:51:15,588 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:51:15,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:51:15,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:51:15,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:51:15,589 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T16:51:15,589 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=814710804, stopped=false 2024-11-15T16:51:15,589 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a9bde5b3a1b5,43823,1731689436438 2024-11-15T16:51:15,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T16:51:15,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T16:51:15,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T16:51:15,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:51:15,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:51:15,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:51:15,591 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T16:51:15,592 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:51:15,592 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T16:51:15,592 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:51:15,592 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:51:15,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:51:15,593 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:51:15,593 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a9bde5b3a1b5,45441,1731689437230' ***** 2024-11-15T16:51:15,593 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T16:51:15,593 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a9bde5b3a1b5,40715,1731689437267' ***** 2024-11-15T16:51:15,593 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T16:51:15,593 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T16:51:15,593 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T16:51:15,593 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T16:51:15,593 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T16:51:15,593 INFO [RS:2;a9bde5b3a1b5:40715 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T16:51:15,594 INFO [RS:2;a9bde5b3a1b5:40715 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T16:51:15,594 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(959): stopping server a9bde5b3a1b5,40715,1731689437267 2024-11-15T16:51:15,594 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T16:51:15,594 INFO [RS:2;a9bde5b3a1b5:40715 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;a9bde5b3a1b5:40715. 2024-11-15T16:51:15,594 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:51:15,594 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:51:15,594 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T16:51:15,594 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T16:51:15,594 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T16:51:15,594 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T16:51:15,594 INFO [RS:1;a9bde5b3a1b5:45441 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T16:51:15,594 INFO [RS:1;a9bde5b3a1b5:45441 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T16:51:15,594 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T16:51:15,594 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-15T16:51:15,594 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(3091): Received CLOSE for bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:51:15,594 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T16:51:15,595 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(959): stopping server a9bde5b3a1b5,45441,1731689437230 2024-11-15T16:51:15,595 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T16:51:15,595 INFO [RS:1;a9bde5b3a1b5:45441 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a9bde5b3a1b5:45441. 2024-11-15T16:51:15,595 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T16:51:15,595 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:51:15,595 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bd339ac2e8d42b7857d656a05e282eea, disabling compactions & flushes 2024-11-15T16:51:15,595 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:51:15,595 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T16:51:15,595 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:51:15,595 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T16:51:15,595 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:51:15,595 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T16:51:15,595 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T16:51:15,595 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T16:51:15,595 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(1325): Online Regions={bd339ac2e8d42b7857d656a05e282eea=testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea.} 2024-11-15T16:51:15,595 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. after waiting 0 ms 2024-11-15T16:51:15,595 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(1351): Waiting on bd339ac2e8d42b7857d656a05e282eea 2024-11-15T16:51:15,595 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:51:15,595 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-15T16:51:15,599 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bd339ac2e8d42b7857d656a05e282eea/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-15T16:51:15,600 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:51:15,600 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bd339ac2e8d42b7857d656a05e282eea: Waiting for close lock at 1731689475595Running coprocessor pre-close hooks at 1731689475595Disabling compacts and flushes for region at 1731689475595Disabling writes for close at 1731689475595Writing region close event to WAL at 1731689475596 (+1 ms)Running coprocessor post-close hooks at 1731689475600 (+4 ms)Closed at 1731689475600 2024-11-15T16:51:15,600 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/.tmp/info/5280c352857b4751bd0f2a4d4e5aee0d is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea./info:regioninfo/1731689455354/Put/seqid=0 2024-11-15T16:51:15,600 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731689451489.bd339ac2e8d42b7857d656a05e282eea. 2024-11-15T16:51:15,603 INFO [regionserver/a9bde5b3a1b5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T16:51:15,603 INFO [regionserver/a9bde5b3a1b5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T16:51:15,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741943_1123 (size=6778) 2024-11-15T16:51:15,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741943_1123 (size=6778) 2024-11-15T16:51:15,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741943_1123 (size=6778) 2024-11-15T16:51:15,608 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/.tmp/info/5280c352857b4751bd0f2a4d4e5aee0d 2024-11-15T16:51:15,614 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/.tmp/info/5280c352857b4751bd0f2a4d4e5aee0d as hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/info/5280c352857b4751bd0f2a4d4e5aee0d 2024-11-15T16:51:15,618 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/info/5280c352857b4751bd0f2a4d4e5aee0d, entries=8, sequenceid=23, filesize=6.6 K 2024-11-15T16:51:15,619 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false 2024-11-15T16:51:15,619 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T16:51:15,625 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-15T16:51:15,625 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T16:51:15,626 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T16:51:15,626 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731689475594Running coprocessor pre-close hooks at 1731689475594Disabling compacts and flushes for region at 1731689475595 (+1 ms)Disabling writes for close at 1731689475595Obtaining lock to block concurrent updates at 1731689475595Preparing flush snapshotting stores in 1588230740 at 1731689475595Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1731689475595Flushing stores of hbase:meta,,1.1588230740 at 1731689475596 (+1 ms)Flushing 1588230740/info: creating writer at 1731689475596Flushing 1588230740/info: appending metadata at 1731689475600 (+4 ms)Flushing 1588230740/info: closing flushed file at 1731689475600Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b09b4c7: reopening flushed file at 1731689475613 (+13 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false at 1731689475619 (+6 ms)Writing region close event to WAL at 1731689475621 (+2 ms)Running coprocessor post-close hooks at 1731689475625 (+4 ms)Closed at 1731689475626 (+1 ms) 2024-11-15T16:51:15,626 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T16:51:15,653 INFO [regionserver/a9bde5b3a1b5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T16:51:15,653 INFO [regionserver/a9bde5b3a1b5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T16:51:15,795 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(976): stopping server a9bde5b3a1b5,40715,1731689437267; all regions closed. 2024-11-15T16:51:15,795 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(976): stopping server a9bde5b3a1b5,45441,1731689437230; all regions closed. 2024-11-15T16:51:15,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741833_1009 (size=2183) 2024-11-15T16:51:15,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741833_1009 (size=2183) 2024-11-15T16:51:15,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741893_1071 (size=1672) 2024-11-15T16:51:15,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741833_1009 (size=2183) 2024-11-15T16:51:15,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741893_1071 (size=1672) 2024-11-15T16:51:15,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741893_1071 (size=1672) 2024-11-15T16:51:15,801 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/oldWALs 2024-11-15T16:51:15,801 INFO [RS:1;a9bde5b3a1b5:45441 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a9bde5b3a1b5%2C45441%2C1731689437230:(num 1731689438717) 2024-11-15T16:51:15,801 DEBUG [RS:1;a9bde5b3a1b5:45441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:51:15,801 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T16:51:15,802 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T16:51:15,802 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.ChoreService(370): Chore service for: regionserver/a9bde5b3a1b5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-15T16:51:15,802 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T16:51:15,802 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T16:51:15,802 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T16:51:15,802 INFO [regionserver/a9bde5b3a1b5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T16:51:15,802 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T16:51:15,802 INFO [RS:1;a9bde5b3a1b5:45441 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45441 2024-11-15T16:51:15,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T16:51:15,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a9bde5b3a1b5,45441,1731689437230 2024-11-15T16:51:15,804 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T16:51:15,804 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/oldWALs 2024-11-15T16:51:15,804 INFO [RS:2;a9bde5b3a1b5:40715 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a9bde5b3a1b5%2C40715%2C1731689437267.meta:.meta(num 1731689454534) 2024-11-15T16:51:15,805 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a9bde5b3a1b5,45441,1731689437230] 2024-11-15T16:51:15,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741834_1010 (size=95) 2024-11-15T16:51:15,807 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a9bde5b3a1b5,45441,1731689437230 already deleted, retry=false 2024-11-15T16:51:15,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741834_1010 (size=95) 2024-11-15T16:51:15,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741834_1010 (size=95) 2024-11-15T16:51:15,807 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a9bde5b3a1b5,45441,1731689437230 expired; onlineServers=1 2024-11-15T16:51:15,810 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/oldWALs 2024-11-15T16:51:15,810 INFO [RS:2;a9bde5b3a1b5:40715 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a9bde5b3a1b5%2C40715%2C1731689437267:(num 1731689438717) 2024-11-15T16:51:15,810 DEBUG [RS:2;a9bde5b3a1b5:40715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:51:15,810 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T16:51:15,810 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T16:51:15,810 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.ChoreService(370): Chore service for: regionserver/a9bde5b3a1b5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-15T16:51:15,810 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T16:51:15,810 INFO [regionserver/a9bde5b3a1b5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T16:51:15,810 INFO [RS:2;a9bde5b3a1b5:40715 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40715 2024-11-15T16:51:15,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a9bde5b3a1b5,40715,1731689437267 2024-11-15T16:51:15,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T16:51:15,812 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T16:51:15,813 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a9bde5b3a1b5,40715,1731689437267] 2024-11-15T16:51:15,815 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a9bde5b3a1b5,40715,1731689437267 already deleted, retry=false 2024-11-15T16:51:15,815 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a9bde5b3a1b5,40715,1731689437267 expired; onlineServers=0 2024-11-15T16:51:15,815 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a9bde5b3a1b5,43823,1731689436438' ***** 2024-11-15T16:51:15,815 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T16:51:15,815 INFO [M:0;a9bde5b3a1b5:43823 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T16:51:15,815 INFO [M:0;a9bde5b3a1b5:43823 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T16:51:15,815 DEBUG [M:0;a9bde5b3a1b5:43823 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T16:51:15,815 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T16:51:15,815 DEBUG [M:0;a9bde5b3a1b5:43823 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T16:51:15,815 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.small.0-1731689438313 {}] cleaner.HFileCleaner(306): Exit Thread[master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.small.0-1731689438313,5,FailOnTimeoutGroup] 2024-11-15T16:51:15,815 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.large.0-1731689438312 {}] cleaner.HFileCleaner(306): Exit Thread[master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.large.0-1731689438312,5,FailOnTimeoutGroup] 2024-11-15T16:51:15,816 INFO [M:0;a9bde5b3a1b5:43823 {}] hbase.ChoreService(370): Chore service for: master/a9bde5b3a1b5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T16:51:15,816 INFO [M:0;a9bde5b3a1b5:43823 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T16:51:15,816 DEBUG [M:0;a9bde5b3a1b5:43823 {}] master.HMaster(1795): Stopping service threads 2024-11-15T16:51:15,816 INFO [M:0;a9bde5b3a1b5:43823 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T16:51:15,816 INFO [M:0;a9bde5b3a1b5:43823 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T16:51:15,817 INFO [M:0;a9bde5b3a1b5:43823 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T16:51:15,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T16:51:15,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:51:15,817 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T16:51:15,817 DEBUG [M:0;a9bde5b3a1b5:43823 {}] zookeeper.ZKUtil(347): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T16:51:15,818 WARN [M:0;a9bde5b3a1b5:43823 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T16:51:15,818 INFO [M:0;a9bde5b3a1b5:43823 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/.lastflushedseqids 2024-11-15T16:51:15,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741944_1124 (size=119) 2024-11-15T16:51:15,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741944_1124 (size=119) 2024-11-15T16:51:15,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741944_1124 (size=119) 2024-11-15T16:51:15,834 INFO [M:0;a9bde5b3a1b5:43823 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T16:51:15,834 INFO [M:0;a9bde5b3a1b5:43823 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T16:51:15,834 DEBUG [M:0;a9bde5b3a1b5:43823 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T16:51:15,834 INFO [M:0;a9bde5b3a1b5:43823 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:51:15,834 DEBUG [M:0;a9bde5b3a1b5:43823 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:51:15,834 DEBUG [M:0;a9bde5b3a1b5:43823 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T16:51:15,834 DEBUG [M:0;a9bde5b3a1b5:43823 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:51:15,834 INFO [M:0;a9bde5b3a1b5:43823 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.38 KB heapSize=102.70 KB 2024-11-15T16:51:15,850 DEBUG [M:0;a9bde5b3a1b5:43823 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dd464863e81f4a09a3f26d409b46e1fb is 82, key is hbase:meta,,1/info:regioninfo/1731689454693/Put/seqid=0 2024-11-15T16:51:15,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741945_1125 (size=6063) 2024-11-15T16:51:15,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741945_1125 (size=6063) 2024-11-15T16:51:15,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741945_1125 (size=6063) 2024-11-15T16:51:15,856 INFO [M:0;a9bde5b3a1b5:43823 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dd464863e81f4a09a3f26d409b46e1fb 2024-11-15T16:51:15,875 DEBUG [M:0;a9bde5b3a1b5:43823 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/84f64081f279471d9d63a3274f54ba8c is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731689451911/Put/seqid=0 2024-11-15T16:51:15,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741946_1126 (size=7906) 2024-11-15T16:51:15,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741946_1126 (size=7906) 2024-11-15T16:51:15,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741946_1126 (size=7906) 2024-11-15T16:51:15,883 INFO [M:0;a9bde5b3a1b5:43823 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.16 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/84f64081f279471d9d63a3274f54ba8c 2024-11-15T16:51:15,887 INFO [M:0;a9bde5b3a1b5:43823 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 84f64081f279471d9d63a3274f54ba8c 2024-11-15T16:51:15,902 DEBUG [M:0;a9bde5b3a1b5:43823 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/782650cb30e64bc99caa4b8c8f652eb1 is 69, key is a9bde5b3a1b5,40715,1731689437267/rs:state/1731689438475/Put/seqid=0 2024-11-15T16:51:15,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741947_1127 (size=5440) 2024-11-15T16:51:15,907 INFO [RS:1;a9bde5b3a1b5:45441 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T16:51:15,907 INFO [RS:1;a9bde5b3a1b5:45441 {}] regionserver.HRegionServer(1031): Exiting; stopping=a9bde5b3a1b5,45441,1731689437230; zookeeper connection closed. 2024-11-15T16:51:15,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:51:15,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45441-0x101acd7477a0002, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:51:15,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741947_1127 (size=5440) 2024-11-15T16:51:15,907 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7218c42 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7218c42 2024-11-15T16:51:15,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741947_1127 (size=5440) 2024-11-15T16:51:15,908 INFO [M:0;a9bde5b3a1b5:43823 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/782650cb30e64bc99caa4b8c8f652eb1 2024-11-15T16:51:15,912 INFO [M:0;a9bde5b3a1b5:43823 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 782650cb30e64bc99caa4b8c8f652eb1 2024-11-15T16:51:15,913 DEBUG [M:0;a9bde5b3a1b5:43823 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dd464863e81f4a09a3f26d409b46e1fb as hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dd464863e81f4a09a3f26d409b46e1fb 2024-11-15T16:51:15,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:51:15,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x101acd7477a0003, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:51:15,913 INFO [RS:2;a9bde5b3a1b5:40715 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T16:51:15,914 INFO [RS:2;a9bde5b3a1b5:40715 {}] regionserver.HRegionServer(1031): Exiting; stopping=a9bde5b3a1b5,40715,1731689437267; zookeeper connection closed. 2024-11-15T16:51:15,914 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@9c7ea66 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@9c7ea66 2024-11-15T16:51:15,914 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-15T16:51:15,917 INFO [M:0;a9bde5b3a1b5:43823 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dd464863e81f4a09a3f26d409b46e1fb, entries=14, sequenceid=207, filesize=5.9 K 2024-11-15T16:51:15,918 DEBUG [M:0;a9bde5b3a1b5:43823 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/84f64081f279471d9d63a3274f54ba8c as hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/84f64081f279471d9d63a3274f54ba8c 2024-11-15T16:51:15,922 INFO [M:0;a9bde5b3a1b5:43823 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 84f64081f279471d9d63a3274f54ba8c 2024-11-15T16:51:15,922 INFO [M:0;a9bde5b3a1b5:43823 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/84f64081f279471d9d63a3274f54ba8c, entries=21, sequenceid=207, filesize=7.7 K 2024-11-15T16:51:15,923 DEBUG [M:0;a9bde5b3a1b5:43823 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/782650cb30e64bc99caa4b8c8f652eb1 as hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/782650cb30e64bc99caa4b8c8f652eb1 2024-11-15T16:51:15,927 INFO [M:0;a9bde5b3a1b5:43823 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 782650cb30e64bc99caa4b8c8f652eb1 2024-11-15T16:51:15,927 INFO [M:0;a9bde5b3a1b5:43823 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40387/user/jenkins/test-data/66bbdeb9-fdd6-f584-8bdd-0f0238df99bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/782650cb30e64bc99caa4b8c8f652eb1, entries=3, sequenceid=207, filesize=5.3 K 2024-11-15T16:51:15,928 INFO [M:0;a9bde5b3a1b5:43823 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 94ms, sequenceid=207, compaction requested=false 2024-11-15T16:51:15,929 INFO [M:0;a9bde5b3a1b5:43823 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:51:15,929 DEBUG [M:0;a9bde5b3a1b5:43823 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731689475834Disabling compacts and flushes for region at 1731689475834Disabling writes for close at 1731689475834Obtaining lock to block concurrent updates at 1731689475834Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731689475834Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85386, getHeapSize=105096, getOffHeapSize=0, getCellsCount=248 at 1731689475834Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731689475835 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731689475835Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731689475849 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731689475850 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731689475861 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731689475875 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731689475875Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731689475888 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731689475901 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731689475901Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17be94e4: reopening flushed file at 1731689475912 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ff96669: reopening flushed file at 1731689475917 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6195643d: reopening flushed file at 1731689475922 (+5 ms)Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 94ms, sequenceid=207, compaction requested=false at 1731689475928 (+6 ms)Writing region close event to WAL at 1731689475929 (+1 ms)Closed at 1731689475929 2024-11-15T16:51:15,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42975 is added to blk_1073741830_1006 (size=69405) 2024-11-15T16:51:15,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741830_1006 (size=69405) 2024-11-15T16:51:15,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39325 is added to blk_1073741830_1006 (size=69405) 2024-11-15T16:51:15,933 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T16:51:15,933 INFO [M:0;a9bde5b3a1b5:43823 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T16:51:15,933 INFO [M:0;a9bde5b3a1b5:43823 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43823 2024-11-15T16:51:15,934 INFO [M:0;a9bde5b3a1b5:43823 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T16:51:16,036 INFO [M:0;a9bde5b3a1b5:43823 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T16:51:16,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:51:16,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43823-0x101acd7477a0000, quorum=127.0.0.1:50974, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T16:51:16,042 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689465588 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689465588 (inode 16688) Holder DFSClient_NONMAPREDUCE_-390475633_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731689456368/wal.1731689465588 (inode 16688) Holder DFSClient_NONMAPREDUCE_-390475633_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-15T16:51:16,045 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689456231 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689456231 (inode 16665) Holder DFSClient_NONMAPREDUCE_-390475633_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731689455556/wal.1731689456231 (inode 16665) Holder DFSClient_NONMAPREDUCE_-390475633_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-15T16:51:16,046 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731689447082/wal.1731689447190 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:51:16,048 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689466321 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689466321 (inode 16714) Holder DFSClient_NONMAPREDUCE_-390475633_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731689465775/wal.1731689466321 (inode 16714) Holder DFSClient_NONMAPREDUCE_-390475633_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-15T16:51:16,048 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731689466474/wal.1731689466533 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:51:16,049 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731689470879/wal.1731689471111 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:51:16,049 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731689440331/wal.1731689440402 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:51:16,051 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal.1731689440200 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal.1731689440200 (inode 16485) Holder DFSClient_NONMAPREDUCE_-390475633_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731689440013/wal.1731689440200 (inode 16485) Holder DFSClient_NONMAPREDUCE_-390475633_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-15T16:51:16,053 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal.1731689439766 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal.1731689439766 (inode 16462) Holder DFSClient_NONMAPREDUCE_-390475633_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731689439501/wal.1731689439766 (inode 16462) Holder DFSClient_NONMAPREDUCE_-390475633_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-15T16:51:16,058 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@43206bef{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T16:51:16,060 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@228ffa29{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T16:51:16,060 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T16:51:16,060 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f079a76{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T16:51:16,061 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7da22a2e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/hadoop.log.dir/,STOPPED} 2024-11-15T16:51:16,063 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T16:51:16,063 WARN [BP-1263208986-172.17.0.2-1731689433335 heartbeating to localhost/127.0.0.1:40387 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T16:51:16,063 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T16:51:16,063 WARN [BP-1263208986-172.17.0.2-1731689433335 heartbeating to localhost/127.0.0.1:40387 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1263208986-172.17.0.2-1731689433335 (Datanode Uuid e2e4cabe-ae6c-4cad-a7b2-f853e118a4ff) service to localhost/127.0.0.1:40387 2024-11-15T16:51:16,065 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data5/current/BP-1263208986-172.17.0.2-1731689433335 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T16:51:16,065 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data6/current/BP-1263208986-172.17.0.2-1731689433335 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T16:51:16,065 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T16:51:16,068 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4546bb60{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T16:51:16,068 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7c7d32f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T16:51:16,068 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T16:51:16,068 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ebbf344{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T16:51:16,068 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b5fc47c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/hadoop.log.dir/,STOPPED} 2024-11-15T16:51:16,070 WARN [BP-1263208986-172.17.0.2-1731689433335 heartbeating to localhost/127.0.0.1:40387 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T16:51:16,070 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T16:51:16,070 WARN [BP-1263208986-172.17.0.2-1731689433335 heartbeating to localhost/127.0.0.1:40387 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1263208986-172.17.0.2-1731689433335 (Datanode Uuid be47a01f-b238-4003-9f9d-9e18c078acd0) service to localhost/127.0.0.1:40387 2024-11-15T16:51:16,070 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T16:51:16,070 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data3/current/BP-1263208986-172.17.0.2-1731689433335 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T16:51:16,071 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data4/current/BP-1263208986-172.17.0.2-1731689433335 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T16:51:16,071 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T16:51:16,073 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@36632d60{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T16:51:16,073 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@751d2fa4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T16:51:16,074 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T16:51:16,074 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@433df981{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T16:51:16,074 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1f76f489{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/hadoop.log.dir/,STOPPED} 2024-11-15T16:51:16,075 WARN [BP-1263208986-172.17.0.2-1731689433335 heartbeating to localhost/127.0.0.1:40387 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T16:51:16,075 WARN [BP-1263208986-172.17.0.2-1731689433335 heartbeating to localhost/127.0.0.1:40387 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1263208986-172.17.0.2-1731689433335 (Datanode Uuid 2db290b3-fd4e-44b9-9c0f-0aaa0bec18d5) service to localhost/127.0.0.1:40387 2024-11-15T16:51:16,076 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T16:51:16,076 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T16:51:16,076 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data1/current/BP-1263208986-172.17.0.2-1731689433335 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T16:51:16,076 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/cluster_6ebfa7a0-da36-d14a-fc27-44b0a0d845ff/data/data2/current/BP-1263208986-172.17.0.2-1731689433335 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T16:51:16,076 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T16:51:16,082 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3717288f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T16:51:16,083 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T16:51:16,083 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T16:51:16,083 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T16:51:16,083 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c91f8e35-3441-d9ac-692b-702dbdb72bb9/hadoop.log.dir/,STOPPED} 2024-11-15T16:51:16,094 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T16:51:16,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down