2024-11-25 07:28:35,905 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-25 07:28:35,916 main DEBUG Took 0.009379 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 07:28:35,917 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 07:28:35,917 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 07:28:35,918 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 07:28:35,919 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:28:35,926 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 07:28:35,937 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:28:35,938 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:28:35,939 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:28:35,939 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:28:35,940 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:28:35,940 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:28:35,941 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:28:35,941 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:28:35,941 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:28:35,942 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:28:35,942 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:28:35,942 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:28:35,943 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:28:35,943 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:28:35,944 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:28:35,944 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:28:35,944 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:28:35,945 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:28:35,945 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:28:35,945 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:28:35,946 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:28:35,946 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:28:35,946 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:28:35,947 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:28:35,947 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:28:35,947 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 07:28:35,949 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:28:35,950 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 07:28:35,952 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 07:28:35,952 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 07:28:35,953 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 07:28:35,953 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 07:28:35,962 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 07:28:35,964 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 07:28:35,965 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 07:28:35,966 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 07:28:35,966 main DEBUG createAppenders(={Console}) 2024-11-25 07:28:35,967 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-11-25 07:28:35,967 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-25 07:28:35,967 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-11-25 07:28:35,968 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 07:28:35,968 main DEBUG OutputStream closed 2024-11-25 07:28:35,968 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 07:28:35,968 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 07:28:35,968 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-11-25 07:28:36,032 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 07:28:36,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 07:28:36,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 07:28:36,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 07:28:36,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 07:28:36,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 07:28:36,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 07:28:36,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 07:28:36,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 07:28:36,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 07:28:36,039 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 07:28:36,039 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 07:28:36,039 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 07:28:36,040 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 07:28:36,040 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 07:28:36,040 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 07:28:36,040 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 07:28:36,041 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 07:28:36,044 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 07:28:36,044 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-11-25 07:28:36,044 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 07:28:36,045 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-11-25T07:28:36,322 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00 2024-11-25 07:28:36,326 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 07:28:36,326 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T07:28:36,337 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-25T07:28:36,368 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T07:28:36,423 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T07:28:36,423 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T07:28:36,437 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-25T07:28:36,455 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65, deleteOnExit=true 2024-11-25T07:28:36,455 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-25T07:28:36,456 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/test.cache.data in system properties and HBase conf 2024-11-25T07:28:36,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/hadoop.tmp.dir in system properties and HBase conf 2024-11-25T07:28:36,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/hadoop.log.dir in system properties and HBase conf 2024-11-25T07:28:36,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-25T07:28:36,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-25T07:28:36,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-25T07:28:36,539 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T07:28:36,634 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T07:28:36,638 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-25T07:28:36,639 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-25T07:28:36,639 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-25T07:28:36,640 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T07:28:36,641 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-25T07:28:36,641 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-25T07:28:36,642 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T07:28:36,643 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T07:28:36,643 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-25T07:28:36,644 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/nfs.dump.dir in system properties and HBase conf 2024-11-25T07:28:36,644 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/java.io.tmpdir in system properties and HBase conf 2024-11-25T07:28:36,645 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T07:28:36,645 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-25T07:28:36,646 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-25T07:28:37,427 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-25T07:28:37,510 INFO [Time-limited test {}] log.Log(170): Logging initialized @2292ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-25T07:28:37,586 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T07:28:37,650 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T07:28:37,679 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T07:28:37,679 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T07:28:37,680 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-25T07:28:37,693 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T07:28:37,695 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751489f9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/hadoop.log.dir/,AVAILABLE} 2024-11-25T07:28:37,696 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7e3a0561{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T07:28:37,888 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@73b32aa2{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/java.io.tmpdir/jetty-localhost-43617-hadoop-hdfs-3_4_1-tests_jar-_-any-4788725243679650335/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T07:28:37,894 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e2ddae{HTTP/1.1, (http/1.1)}{localhost:43617} 2024-11-25T07:28:37,895 INFO [Time-limited test {}] server.Server(415): Started @2677ms 2024-11-25T07:28:38,267 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T07:28:38,273 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T07:28:38,274 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T07:28:38,275 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T07:28:38,275 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T07:28:38,276 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@50901468{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/hadoop.log.dir/,AVAILABLE} 2024-11-25T07:28:38,276 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@19895485{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T07:28:38,395 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@17be2a4f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/java.io.tmpdir/jetty-localhost-42469-hadoop-hdfs-3_4_1-tests_jar-_-any-3168919043887046713/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T07:28:38,396 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2ecf726e{HTTP/1.1, (http/1.1)}{localhost:42469} 2024-11-25T07:28:38,396 INFO [Time-limited test {}] server.Server(415): Started @3178ms 2024-11-25T07:28:38,449 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T07:28:38,584 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T07:28:38,588 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T07:28:38,593 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T07:28:38,593 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T07:28:38,593 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T07:28:38,594 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b152b3a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/hadoop.log.dir/,AVAILABLE} 2024-11-25T07:28:38,595 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4edd15fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T07:28:38,717 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@70b25848{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/java.io.tmpdir/jetty-localhost-38553-hadoop-hdfs-3_4_1-tests_jar-_-any-11186213074430271066/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T07:28:38,718 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6a8a49ff{HTTP/1.1, (http/1.1)}{localhost:38553} 2024-11-25T07:28:38,718 INFO [Time-limited test {}] server.Server(415): Started @3500ms 2024-11-25T07:28:38,720 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T07:28:38,761 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T07:28:38,767 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T07:28:38,770 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T07:28:38,771 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T07:28:38,771 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T07:28:38,772 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7bd4c69c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/hadoop.log.dir/,AVAILABLE} 2024-11-25T07:28:38,773 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@582ba343{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T07:28:38,895 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1c5b5cda{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/java.io.tmpdir/jetty-localhost-35899-hadoop-hdfs-3_4_1-tests_jar-_-any-10183677178651517803/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T07:28:38,896 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@32eb964{HTTP/1.1, (http/1.1)}{localhost:35899} 2024-11-25T07:28:38,896 INFO [Time-limited test {}] server.Server(415): Started @3678ms 2024-11-25T07:28:38,898 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T07:28:38,943 WARN [Thread-115 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data3/current/BP-661403191-172.17.0.2-1732519717195/current, will proceed with Du for space computation calculation, 2024-11-25T07:28:38,943 WARN [Thread-114 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data1/current/BP-661403191-172.17.0.2-1732519717195/current, will proceed with Du for space computation calculation, 2024-11-25T07:28:38,943 WARN [Thread-118 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data4/current/BP-661403191-172.17.0.2-1732519717195/current, will proceed with Du for space computation calculation, 2024-11-25T07:28:38,943 WARN [Thread-117 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data2/current/BP-661403191-172.17.0.2-1732519717195/current, will proceed with Du for space computation calculation, 2024-11-25T07:28:39,003 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T07:28:39,003 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T07:28:39,050 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data5/current/BP-661403191-172.17.0.2-1732519717195/current, will proceed with Du for space computation calculation, 2024-11-25T07:28:39,051 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data6/current/BP-661403191-172.17.0.2-1732519717195/current, will proceed with Du for space computation calculation, 2024-11-25T07:28:39,083 WARN [Thread-111 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T07:28:39,086 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf6a8cde0db905b89 with lease ID 0xabf718327ac697bf: Processing first storage report for DS-533c0c67-8243-4b60-b744-302e9d9fc36e from datanode DatanodeRegistration(127.0.0.1:37043, datanodeUuid=48ca7e09-c38c-495c-bc02-27c692a838eb, infoPort=33427, infoSecurePort=0, ipcPort=37307, storageInfo=lv=-57;cid=testClusterID;nsid=1232047988;c=1732519717195) 2024-11-25T07:28:39,087 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf6a8cde0db905b89 with lease ID 0xabf718327ac697bf: from storage DS-533c0c67-8243-4b60-b744-302e9d9fc36e node DatanodeRegistration(127.0.0.1:37043, datanodeUuid=48ca7e09-c38c-495c-bc02-27c692a838eb, infoPort=33427, infoSecurePort=0, ipcPort=37307, storageInfo=lv=-57;cid=testClusterID;nsid=1232047988;c=1732519717195), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-25T07:28:39,088 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x491307e63b47c039 with lease ID 0xabf718327ac697c0: Processing first storage report for DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc from datanode DatanodeRegistration(127.0.0.1:37227, datanodeUuid=c7b23268-a363-4476-9499-b348c306f2b9, infoPort=43429, infoSecurePort=0, ipcPort=41097, storageInfo=lv=-57;cid=testClusterID;nsid=1232047988;c=1732519717195) 2024-11-25T07:28:39,088 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x491307e63b47c039 with lease ID 0xabf718327ac697c0: from storage DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc node DatanodeRegistration(127.0.0.1:37227, datanodeUuid=c7b23268-a363-4476-9499-b348c306f2b9, infoPort=43429, infoSecurePort=0, ipcPort=41097, storageInfo=lv=-57;cid=testClusterID;nsid=1232047988;c=1732519717195), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-25T07:28:39,088 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf6a8cde0db905b89 with lease ID 0xabf718327ac697bf: Processing first storage report for DS-17e09bc5-6f58-45dd-946e-859b07263fad from datanode DatanodeRegistration(127.0.0.1:37043, datanodeUuid=48ca7e09-c38c-495c-bc02-27c692a838eb, infoPort=33427, infoSecurePort=0, ipcPort=37307, storageInfo=lv=-57;cid=testClusterID;nsid=1232047988;c=1732519717195) 2024-11-25T07:28:39,088 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf6a8cde0db905b89 with lease ID 0xabf718327ac697bf: from storage DS-17e09bc5-6f58-45dd-946e-859b07263fad node DatanodeRegistration(127.0.0.1:37043, datanodeUuid=48ca7e09-c38c-495c-bc02-27c692a838eb, infoPort=33427, infoSecurePort=0, ipcPort=37307, storageInfo=lv=-57;cid=testClusterID;nsid=1232047988;c=1732519717195), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T07:28:39,089 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x491307e63b47c039 with lease ID 0xabf718327ac697c0: Processing first storage report for DS-67343dc9-85ca-42e1-8a06-625fd24dd803 from datanode DatanodeRegistration(127.0.0.1:37227, datanodeUuid=c7b23268-a363-4476-9499-b348c306f2b9, infoPort=43429, infoSecurePort=0, ipcPort=41097, storageInfo=lv=-57;cid=testClusterID;nsid=1232047988;c=1732519717195) 2024-11-25T07:28:39,089 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x491307e63b47c039 with lease ID 0xabf718327ac697c0: from storage DS-67343dc9-85ca-42e1-8a06-625fd24dd803 node DatanodeRegistration(127.0.0.1:37227, datanodeUuid=c7b23268-a363-4476-9499-b348c306f2b9, infoPort=43429, infoSecurePort=0, ipcPort=41097, storageInfo=lv=-57;cid=testClusterID;nsid=1232047988;c=1732519717195), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T07:28:39,091 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x38a29c11232c7a3a with lease ID 0xabf718327ac697c1: Processing first storage report for DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18 from datanode DatanodeRegistration(127.0.0.1:33603, datanodeUuid=f75da14a-acc5-4dbe-9081-d37c01774a00, infoPort=37263, infoSecurePort=0, ipcPort=34443, storageInfo=lv=-57;cid=testClusterID;nsid=1232047988;c=1732519717195) 2024-11-25T07:28:39,091 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x38a29c11232c7a3a with lease ID 0xabf718327ac697c1: from storage DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18 node DatanodeRegistration(127.0.0.1:33603, datanodeUuid=f75da14a-acc5-4dbe-9081-d37c01774a00, infoPort=37263, infoSecurePort=0, ipcPort=34443, storageInfo=lv=-57;cid=testClusterID;nsid=1232047988;c=1732519717195), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T07:28:39,092 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x38a29c11232c7a3a with lease ID 0xabf718327ac697c1: Processing first storage report for DS-1a20e8fd-ba99-48fa-a60e-4d26e14699fb from datanode DatanodeRegistration(127.0.0.1:33603, datanodeUuid=f75da14a-acc5-4dbe-9081-d37c01774a00, infoPort=37263, infoSecurePort=0, ipcPort=34443, storageInfo=lv=-57;cid=testClusterID;nsid=1232047988;c=1732519717195) 2024-11-25T07:28:39,092 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x38a29c11232c7a3a with lease ID 0xabf718327ac697c1: from storage DS-1a20e8fd-ba99-48fa-a60e-4d26e14699fb node DatanodeRegistration(127.0.0.1:33603, datanodeUuid=f75da14a-acc5-4dbe-9081-d37c01774a00, infoPort=37263, infoSecurePort=0, ipcPort=34443, storageInfo=lv=-57;cid=testClusterID;nsid=1232047988;c=1732519717195), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T07:28:39,278 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00 2024-11-25T07:28:39,385 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/zookeeper_0, clientPort=50797, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T07:28:39,395 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50797 2024-11-25T07:28:39,405 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:28:39,407 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:28:39,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741825_1001 (size=7) 2024-11-25T07:28:39,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741825_1001 (size=7) 2024-11-25T07:28:39,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741825_1001 (size=7) 2024-11-25T07:28:40,014 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438 with version=8 2024-11-25T07:28:40,014 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/hbase-staging 2024-11-25T07:28:40,317 INFO [Time-limited test {}] client.ConnectionUtils(128): master/5eb3d201e8c9:0 server-side Connection retries=45 2024-11-25T07:28:40,328 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T07:28:40,328 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T07:28:40,333 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T07:28:40,333 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T07:28:40,333 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T07:28:40,471 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T07:28:40,533 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T07:28:40,542 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T07:28:40,546 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T07:28:40,574 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 21037 (auto-detected) 2024-11-25T07:28:40,575 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-25T07:28:40,595 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33903 2024-11-25T07:28:40,622 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33903 connecting to ZooKeeper ensemble=127.0.0.1:50797 2024-11-25T07:28:40,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:339030x0, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T07:28:40,658 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33903-0x1014e06dc560000 connected 2024-11-25T07:28:40,686 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:28:40,689 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:28:40,699 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T07:28:40,703 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438, hbase.cluster.distributed=false 2024-11-25T07:28:40,726 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T07:28:40,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33903 2024-11-25T07:28:40,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33903 2024-11-25T07:28:40,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33903 2024-11-25T07:28:40,735 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33903 2024-11-25T07:28:40,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33903 2024-11-25T07:28:40,855 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5eb3d201e8c9:0 server-side Connection retries=45 2024-11-25T07:28:40,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T07:28:40,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T07:28:40,857 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T07:28:40,858 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T07:28:40,858 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T07:28:40,861 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T07:28:40,863 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T07:28:40,864 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33753 2024-11-25T07:28:40,867 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33753 connecting to ZooKeeper ensemble=127.0.0.1:50797 2024-11-25T07:28:40,868 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:28:40,872 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:28:40,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:337530x0, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T07:28:40,880 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33753-0x1014e06dc560001 connected 2024-11-25T07:28:40,880 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T07:28:40,884 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T07:28:40,892 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T07:28:40,895 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T07:28:40,899 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T07:28:40,900 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33753 2024-11-25T07:28:40,900 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33753 2024-11-25T07:28:40,901 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33753 2024-11-25T07:28:40,901 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33753 2024-11-25T07:28:40,902 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33753 2024-11-25T07:28:40,918 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5eb3d201e8c9:0 server-side Connection retries=45 2024-11-25T07:28:40,919 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T07:28:40,919 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T07:28:40,919 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T07:28:40,919 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T07:28:40,920 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T07:28:40,920 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T07:28:40,920 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T07:28:40,921 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44311 2024-11-25T07:28:40,924 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44311 connecting to ZooKeeper ensemble=127.0.0.1:50797 2024-11-25T07:28:40,925 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:28:40,930 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:28:40,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:443110x0, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T07:28:40,942 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44311-0x1014e06dc560002 connected 2024-11-25T07:28:40,942 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T07:28:40,943 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T07:28:40,944 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T07:28:40,945 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T07:28:40,947 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T07:28:40,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44311 2024-11-25T07:28:40,952 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44311 2024-11-25T07:28:40,953 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44311 2024-11-25T07:28:40,953 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44311 2024-11-25T07:28:40,954 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44311 2024-11-25T07:28:40,972 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5eb3d201e8c9:0 server-side Connection retries=45 2024-11-25T07:28:40,973 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T07:28:40,973 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T07:28:40,973 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T07:28:40,973 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T07:28:40,973 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T07:28:40,974 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T07:28:40,974 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T07:28:40,975 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35771 2024-11-25T07:28:40,977 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35771 connecting to ZooKeeper ensemble=127.0.0.1:50797 2024-11-25T07:28:40,978 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:28:40,980 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:28:40,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:357710x0, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T07:28:40,988 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:357710x0, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T07:28:40,988 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35771-0x1014e06dc560003 connected 2024-11-25T07:28:40,988 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T07:28:40,989 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T07:28:40,990 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35771-0x1014e06dc560003, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T07:28:40,992 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35771-0x1014e06dc560003, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T07:28:40,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35771 2024-11-25T07:28:40,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35771 2024-11-25T07:28:40,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35771 2024-11-25T07:28:40,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35771 2024-11-25T07:28:41,000 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35771 2024-11-25T07:28:41,015 DEBUG [M:0;5eb3d201e8c9:33903 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;5eb3d201e8c9:33903 2024-11-25T07:28:41,016 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/5eb3d201e8c9,33903,1732519720118 2024-11-25T07:28:41,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35771-0x1014e06dc560003, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T07:28:41,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T07:28:41,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T07:28:41,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T07:28:41,027 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/5eb3d201e8c9,33903,1732519720118 2024-11-25T07:28:41,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35771-0x1014e06dc560003, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T07:28:41,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T07:28:41,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35771-0x1014e06dc560003, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T07:28:41,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,050 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T07:28:41,052 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/5eb3d201e8c9,33903,1732519720118 from backup master directory 2024-11-25T07:28:41,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/5eb3d201e8c9,33903,1732519720118 2024-11-25T07:28:41,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T07:28:41,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35771-0x1014e06dc560003, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T07:28:41,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T07:28:41,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T07:28:41,056 WARN [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T07:28:41,056 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=5eb3d201e8c9,33903,1732519720118 2024-11-25T07:28:41,058 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T07:28:41,059 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T07:28:41,117 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/hbase.id] with ID: f004f047-221d-470c-b151-81737d8f21a0 2024-11-25T07:28:41,117 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/.tmp/hbase.id 2024-11-25T07:28:41,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741826_1002 (size=42) 2024-11-25T07:28:41,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741826_1002 (size=42) 2024-11-25T07:28:41,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741826_1002 (size=42) 2024-11-25T07:28:41,134 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/.tmp/hbase.id]:[hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/hbase.id] 2024-11-25T07:28:41,179 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:28:41,185 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T07:28:41,205 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-25T07:28:41,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35771-0x1014e06dc560003, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741827_1003 (size=196) 2024-11-25T07:28:41,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741827_1003 (size=196) 2024-11-25T07:28:41,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741827_1003 (size=196) 2024-11-25T07:28:41,245 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T07:28:41,247 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T07:28:41,253 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:41,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741828_1004 (size=1189) 2024-11-25T07:28:41,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741828_1004 (size=1189) 2024-11-25T07:28:41,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741828_1004 (size=1189) 2024-11-25T07:28:41,308 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store 2024-11-25T07:28:41,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741829_1005 (size=34) 2024-11-25T07:28:41,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741829_1005 (size=34) 2024-11-25T07:28:41,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741829_1005 (size=34) 2024-11-25T07:28:41,339 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T07:28:41,342 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:41,344 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T07:28:41,344 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T07:28:41,345 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T07:28:41,347 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T07:28:41,347 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T07:28:41,347 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T07:28:41,348 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732519721344Disabling compacts and flushes for region at 1732519721344Disabling writes for close at 1732519721347 (+3 ms)Writing region close event to WAL at 1732519721347Closed at 1732519721347 2024-11-25T07:28:41,351 WARN [master/5eb3d201e8c9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/.initializing 2024-11-25T07:28:41,351 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/WALs/5eb3d201e8c9,33903,1732519720118 2024-11-25T07:28:41,359 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T07:28:41,373 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5eb3d201e8c9%2C33903%2C1732519720118, suffix=, logDir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/WALs/5eb3d201e8c9,33903,1732519720118, archiveDir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/oldWALs, maxLogs=10 2024-11-25T07:28:41,402 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/WALs/5eb3d201e8c9,33903,1732519720118/5eb3d201e8c9%2C33903%2C1732519720118.1732519721378, exclude list is [], retry=0 2024-11-25T07:28:41,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:28:41,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:41,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:41,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:41,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-25T07:28:41,469 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/WALs/5eb3d201e8c9,33903,1732519720118/5eb3d201e8c9%2C33903%2C1732519720118.1732519721378 2024-11-25T07:28:41,470 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:37263:37263)] 2024-11-25T07:28:41,471 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:28:41,471 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:41,474 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:28:41,476 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:28:41,514 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:28:41,540 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T07:28:41,543 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:41,546 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:28:41,547 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:28:41,549 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T07:28:41,550 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:41,551 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:41,551 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:28:41,553 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T07:28:41,553 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:41,554 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:41,554 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:28:41,557 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T07:28:41,557 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:41,557 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:41,558 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:28:41,561 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:28:41,562 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:28:41,567 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:28:41,567 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:28:41,570 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T07:28:41,573 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:28:41,577 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:28:41,578 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66130593, jitterRate=-0.01457737386226654}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T07:28:41,584 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732519721489Initializing all the Stores at 1732519721491 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732519721491Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519721492 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519721492Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519721492Cleaning up temporary data from old regions at 1732519721567 (+75 ms)Region opened successfully at 1732519721584 (+17 ms) 2024-11-25T07:28:41,585 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T07:28:41,619 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e571963, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5eb3d201e8c9/172.17.0.2:0 2024-11-25T07:28:41,652 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T07:28:41,664 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T07:28:41,664 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T07:28:41,667 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T07:28:41,668 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-25T07:28:41,673 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-25T07:28:41,674 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T07:28:41,699 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T07:28:41,706 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T07:28:41,710 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T07:28:41,712 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T07:28:41,714 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T07:28:41,715 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T07:28:41,717 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T07:28:41,720 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T07:28:41,722 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T07:28:41,723 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T07:28:41,724 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T07:28:41,742 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T07:28:41,744 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T07:28:41,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35771-0x1014e06dc560003, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T07:28:41,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T07:28:41,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T07:28:41,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T07:28:41,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35771-0x1014e06dc560003, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,751 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=5eb3d201e8c9,33903,1732519720118, sessionid=0x1014e06dc560000, setting cluster-up flag (Was=false) 2024-11-25T07:28:41,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35771-0x1014e06dc560003, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,770 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T07:28:41,771 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5eb3d201e8c9,33903,1732519720118 2024-11-25T07:28:41,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35771-0x1014e06dc560003, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:41,783 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T07:28:41,784 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5eb3d201e8c9,33903,1732519720118 2024-11-25T07:28:41,790 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T07:28:41,804 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(746): ClusterId : f004f047-221d-470c-b151-81737d8f21a0 2024-11-25T07:28:41,804 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(746): ClusterId : f004f047-221d-470c-b151-81737d8f21a0 2024-11-25T07:28:41,804 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.HRegionServer(746): ClusterId : f004f047-221d-470c-b151-81737d8f21a0 2024-11-25T07:28:41,807 DEBUG [RS:0;5eb3d201e8c9:33753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T07:28:41,807 DEBUG [RS:1;5eb3d201e8c9:44311 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T07:28:41,807 DEBUG [RS:2;5eb3d201e8c9:35771 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T07:28:41,813 DEBUG [RS:0;5eb3d201e8c9:33753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T07:28:41,813 DEBUG [RS:2;5eb3d201e8c9:35771 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T07:28:41,813 DEBUG [RS:1;5eb3d201e8c9:44311 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T07:28:41,813 DEBUG [RS:0;5eb3d201e8c9:33753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T07:28:41,813 DEBUG [RS:1;5eb3d201e8c9:44311 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T07:28:41,813 DEBUG [RS:2;5eb3d201e8c9:35771 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T07:28:41,816 DEBUG [RS:0;5eb3d201e8c9:33753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T07:28:41,816 DEBUG [RS:2;5eb3d201e8c9:35771 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T07:28:41,816 DEBUG [RS:1;5eb3d201e8c9:44311 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T07:28:41,817 DEBUG [RS:2;5eb3d201e8c9:35771 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@307b98a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5eb3d201e8c9/172.17.0.2:0 2024-11-25T07:28:41,817 DEBUG [RS:0;5eb3d201e8c9:33753 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1304ad04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5eb3d201e8c9/172.17.0.2:0 2024-11-25T07:28:41,817 DEBUG [RS:1;5eb3d201e8c9:44311 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a19730c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5eb3d201e8c9/172.17.0.2:0 2024-11-25T07:28:41,833 DEBUG [RS:0;5eb3d201e8c9:33753 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;5eb3d201e8c9:33753 2024-11-25T07:28:41,833 DEBUG [RS:2;5eb3d201e8c9:35771 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;5eb3d201e8c9:35771 2024-11-25T07:28:41,836 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T07:28:41,836 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T07:28:41,836 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T07:28:41,836 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T07:28:41,836 DEBUG [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T07:28:41,836 DEBUG [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T07:28:41,838 DEBUG [RS:1;5eb3d201e8c9:44311 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;5eb3d201e8c9:44311 2024-11-25T07:28:41,838 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T07:28:41,838 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T07:28:41,838 DEBUG [RS:1;5eb3d201e8c9:44311 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T07:28:41,839 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(2659): reportForDuty to master=5eb3d201e8c9,33903,1732519720118 with port=33753, startcode=1732519720813 2024-11-25T07:28:41,839 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(2659): reportForDuty to master=5eb3d201e8c9,33903,1732519720118 with port=35771, startcode=1732519720972 2024-11-25T07:28:41,839 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.HRegionServer(2659): reportForDuty to master=5eb3d201e8c9,33903,1732519720118 with port=44311, startcode=1732519720918 2024-11-25T07:28:41,851 DEBUG [RS:2;5eb3d201e8c9:35771 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T07:28:41,851 DEBUG [RS:0;5eb3d201e8c9:33753 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T07:28:41,851 DEBUG [RS:1;5eb3d201e8c9:44311 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T07:28:41,888 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58019, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T07:28:41,888 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60931, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T07:28:41,888 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48197, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T07:28:41,888 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T07:28:41,894 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33903 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T07:28:41,897 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T07:28:41,899 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33903 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T07:28:41,900 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33903 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T07:28:41,904 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T07:28:41,909 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 5eb3d201e8c9,33903,1732519720118 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T07:28:41,916 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/5eb3d201e8c9:0, corePoolSize=5, maxPoolSize=5 2024-11-25T07:28:41,916 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/5eb3d201e8c9:0, corePoolSize=5, maxPoolSize=5 2024-11-25T07:28:41,916 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/5eb3d201e8c9:0, corePoolSize=5, maxPoolSize=5 2024-11-25T07:28:41,916 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/5eb3d201e8c9:0, corePoolSize=5, maxPoolSize=5 2024-11-25T07:28:41,916 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/5eb3d201e8c9:0, corePoolSize=10, maxPoolSize=10 2024-11-25T07:28:41,916 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:41,916 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/5eb3d201e8c9:0, corePoolSize=2, maxPoolSize=2 2024-11-25T07:28:41,916 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:41,918 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732519751918 2024-11-25T07:28:41,920 DEBUG [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-25T07:28:41,920 DEBUG [RS:1;5eb3d201e8c9:44311 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-25T07:28:41,920 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T07:28:41,920 DEBUG [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-25T07:28:41,920 WARN [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-25T07:28:41,920 WARN [RS:1;5eb3d201e8c9:44311 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-25T07:28:41,920 WARN [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-25T07:28:41,921 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T07:28:41,921 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T07:28:41,922 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T07:28:41,924 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T07:28:41,925 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T07:28:41,925 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T07:28:41,925 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T07:28:41,926 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:41,927 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:41,928 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T07:28:41,928 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T07:28:41,929 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T07:28:41,930 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T07:28:41,932 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T07:28:41,932 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T07:28:41,934 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/5eb3d201e8c9:0:becomeActiveMaster-HFileCleaner.large.0-1732519721933,5,FailOnTimeoutGroup] 2024-11-25T07:28:41,934 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/5eb3d201e8c9:0:becomeActiveMaster-HFileCleaner.small.0-1732519721934,5,FailOnTimeoutGroup] 2024-11-25T07:28:41,935 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:41,935 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T07:28:41,936 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:41,936 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:41,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741831_1007 (size=1321) 2024-11-25T07:28:41,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741831_1007 (size=1321) 2024-11-25T07:28:41,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741831_1007 (size=1321) 2024-11-25T07:28:41,962 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T07:28:41,962 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438 2024-11-25T07:28:41,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741832_1008 (size=32) 2024-11-25T07:28:41,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741832_1008 (size=32) 2024-11-25T07:28:41,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741832_1008 (size=32) 2024-11-25T07:28:41,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:41,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T07:28:41,979 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T07:28:41,980 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:41,980 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:28:41,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T07:28:41,982 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T07:28:41,983 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:41,983 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:28:41,983 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T07:28:41,985 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T07:28:41,985 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:41,986 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:28:41,986 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T07:28:41,988 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T07:28:41,988 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:41,989 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:28:41,989 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T07:28:41,991 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740 2024-11-25T07:28:41,991 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740 2024-11-25T07:28:41,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T07:28:41,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T07:28:41,995 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T07:28:41,997 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T07:28:42,000 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:28:42,001 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62666945, jitterRate=-0.06618975102901459}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T07:28:42,004 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732519721975Initializing all the Stores at 1732519721977 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732519721977Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732519721977Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519721977Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732519721977Cleaning up temporary data from old regions at 1732519721994 (+17 ms)Region opened successfully at 1732519722004 (+10 ms) 2024-11-25T07:28:42,004 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T07:28:42,004 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T07:28:42,004 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T07:28:42,004 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T07:28:42,004 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T07:28:42,006 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T07:28:42,006 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732519722004Disabling compacts and flushes for region at 1732519722004Disabling writes for close at 1732519722004Writing region close event to WAL at 1732519722005 (+1 ms)Closed at 1732519722006 (+1 ms) 2024-11-25T07:28:42,009 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T07:28:42,009 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T07:28:42,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T07:28:42,021 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.HRegionServer(2659): reportForDuty to master=5eb3d201e8c9,33903,1732519720118 with port=44311, startcode=1732519720918 2024-11-25T07:28:42,021 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(2659): reportForDuty to master=5eb3d201e8c9,33903,1732519720118 with port=35771, startcode=1732519720972 2024-11-25T07:28:42,021 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(2659): reportForDuty to master=5eb3d201e8c9,33903,1732519720118 with port=33753, startcode=1732519720813 2024-11-25T07:28:42,023 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33903 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5eb3d201e8c9,35771,1732519720972 2024-11-25T07:28:42,025 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T07:28:42,026 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33903 {}] master.ServerManager(517): Registering regionserver=5eb3d201e8c9,35771,1732519720972 2024-11-25T07:28:42,028 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T07:28:42,036 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33903 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5eb3d201e8c9,44311,1732519720918 2024-11-25T07:28:42,036 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33903 {}] master.ServerManager(517): Registering regionserver=5eb3d201e8c9,44311,1732519720918 2024-11-25T07:28:42,036 DEBUG [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438 2024-11-25T07:28:42,036 DEBUG [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44261 2024-11-25T07:28:42,036 DEBUG [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T07:28:42,039 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33903 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5eb3d201e8c9,33753,1732519720813 2024-11-25T07:28:42,039 DEBUG [RS:1;5eb3d201e8c9:44311 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438 2024-11-25T07:28:42,039 DEBUG [RS:1;5eb3d201e8c9:44311 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44261 2024-11-25T07:28:42,039 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33903 {}] master.ServerManager(517): Registering regionserver=5eb3d201e8c9,33753,1732519720813 2024-11-25T07:28:42,039 DEBUG [RS:1;5eb3d201e8c9:44311 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T07:28:42,042 DEBUG [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438 2024-11-25T07:28:42,042 DEBUG [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44261 2024-11-25T07:28:42,042 DEBUG [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T07:28:42,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T07:28:42,046 DEBUG [RS:2;5eb3d201e8c9:35771 {}] zookeeper.ZKUtil(111): regionserver:35771-0x1014e06dc560003, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5eb3d201e8c9,35771,1732519720972 2024-11-25T07:28:42,046 DEBUG [RS:1;5eb3d201e8c9:44311 {}] zookeeper.ZKUtil(111): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5eb3d201e8c9,44311,1732519720918 2024-11-25T07:28:42,047 WARN [RS:2;5eb3d201e8c9:35771 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T07:28:42,047 WARN [RS:1;5eb3d201e8c9:44311 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T07:28:42,047 INFO [RS:2;5eb3d201e8c9:35771 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:42,047 INFO [RS:1;5eb3d201e8c9:44311 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:42,047 DEBUG [RS:0;5eb3d201e8c9:33753 {}] zookeeper.ZKUtil(111): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5eb3d201e8c9,33753,1732519720813 2024-11-25T07:28:42,047 WARN [RS:0;5eb3d201e8c9:33753 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T07:28:42,047 INFO [RS:0;5eb3d201e8c9:33753 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:42,047 DEBUG [RS:1;5eb3d201e8c9:44311 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,44311,1732519720918 2024-11-25T07:28:42,047 DEBUG [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972 2024-11-25T07:28:42,047 DEBUG [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,33753,1732519720813 2024-11-25T07:28:42,048 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5eb3d201e8c9,44311,1732519720918] 2024-11-25T07:28:42,049 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5eb3d201e8c9,33753,1732519720813] 2024-11-25T07:28:42,049 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5eb3d201e8c9,35771,1732519720972] 2024-11-25T07:28:42,074 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T07:28:42,074 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T07:28:42,074 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T07:28:42,087 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T07:28:42,087 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T07:28:42,088 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T07:28:42,093 INFO [RS:0;5eb3d201e8c9:33753 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T07:28:42,093 INFO [RS:1;5eb3d201e8c9:44311 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T07:28:42,093 INFO [RS:2;5eb3d201e8c9:35771 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T07:28:42,094 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,094 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,094 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,095 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T07:28:42,099 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T07:28:42,100 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T07:28:42,101 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T07:28:42,101 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T07:28:42,101 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T07:28:42,103 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,103 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,103 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,103 DEBUG [RS:1;5eb3d201e8c9:44311 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,103 DEBUG [RS:0;5eb3d201e8c9:33753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,103 DEBUG [RS:2;5eb3d201e8c9:35771 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,103 DEBUG [RS:1;5eb3d201e8c9:44311 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,103 DEBUG [RS:2;5eb3d201e8c9:35771 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,103 DEBUG [RS:0;5eb3d201e8c9:33753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,103 DEBUG [RS:1;5eb3d201e8c9:44311 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,103 DEBUG [RS:2;5eb3d201e8c9:35771 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,103 DEBUG [RS:0;5eb3d201e8c9:33753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,103 DEBUG [RS:1;5eb3d201e8c9:44311 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,103 DEBUG [RS:2;5eb3d201e8c9:35771 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,104 DEBUG [RS:2;5eb3d201e8c9:35771 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,104 DEBUG [RS:0;5eb3d201e8c9:33753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,104 DEBUG [RS:1;5eb3d201e8c9:44311 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,104 DEBUG [RS:2;5eb3d201e8c9:35771 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5eb3d201e8c9:0, corePoolSize=2, maxPoolSize=2 2024-11-25T07:28:42,104 DEBUG [RS:0;5eb3d201e8c9:33753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,104 DEBUG [RS:2;5eb3d201e8c9:35771 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,104 DEBUG [RS:1;5eb3d201e8c9:44311 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5eb3d201e8c9:0, corePoolSize=2, maxPoolSize=2 2024-11-25T07:28:42,104 DEBUG [RS:2;5eb3d201e8c9:35771 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,104 DEBUG [RS:0;5eb3d201e8c9:33753 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5eb3d201e8c9:0, corePoolSize=2, maxPoolSize=2 2024-11-25T07:28:42,104 DEBUG [RS:1;5eb3d201e8c9:44311 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,104 DEBUG [RS:2;5eb3d201e8c9:35771 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,104 DEBUG [RS:0;5eb3d201e8c9:33753 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,104 DEBUG [RS:2;5eb3d201e8c9:35771 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,104 DEBUG [RS:1;5eb3d201e8c9:44311 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,104 DEBUG [RS:2;5eb3d201e8c9:35771 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,104 DEBUG [RS:0;5eb3d201e8c9:33753 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,104 DEBUG [RS:1;5eb3d201e8c9:44311 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,105 DEBUG [RS:2;5eb3d201e8c9:35771 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,105 DEBUG [RS:0;5eb3d201e8c9:33753 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,105 DEBUG [RS:2;5eb3d201e8c9:35771 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0, corePoolSize=3, maxPoolSize=3 2024-11-25T07:28:42,105 DEBUG [RS:1;5eb3d201e8c9:44311 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,105 DEBUG [RS:0;5eb3d201e8c9:33753 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,105 DEBUG [RS:2;5eb3d201e8c9:35771 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5eb3d201e8c9:0, corePoolSize=3, maxPoolSize=3 2024-11-25T07:28:42,105 DEBUG [RS:1;5eb3d201e8c9:44311 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,105 DEBUG [RS:0;5eb3d201e8c9:33753 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,105 DEBUG [RS:1;5eb3d201e8c9:44311 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,105 DEBUG [RS:0;5eb3d201e8c9:33753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:28:42,105 DEBUG [RS:1;5eb3d201e8c9:44311 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0, corePoolSize=3, maxPoolSize=3 2024-11-25T07:28:42,105 DEBUG [RS:0;5eb3d201e8c9:33753 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0, corePoolSize=3, maxPoolSize=3 2024-11-25T07:28:42,105 DEBUG [RS:1;5eb3d201e8c9:44311 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5eb3d201e8c9:0, corePoolSize=3, maxPoolSize=3 2024-11-25T07:28:42,106 DEBUG [RS:0;5eb3d201e8c9:33753 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5eb3d201e8c9:0, corePoolSize=3, maxPoolSize=3 2024-11-25T07:28:42,107 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,107 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,107 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,107 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,107 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,108 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,35771,1732519720972-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T07:28:42,108 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,108 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,108 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,108 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,108 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,108 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,108 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,108 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,109 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,109 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,109 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,44311,1732519720918-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T07:28:42,109 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,33753,1732519720813-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T07:28:42,140 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T07:28:42,140 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T07:28:42,140 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T07:28:42,143 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,44311,1732519720918-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,143 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,35771,1732519720972-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,143 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,33753,1732519720813-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,143 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,143 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,143 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,143 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.Replication(171): 5eb3d201e8c9,33753,1732519720813 started 2024-11-25T07:28:42,143 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.Replication(171): 5eb3d201e8c9,35771,1732519720972 started 2024-11-25T07:28:42,143 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.Replication(171): 5eb3d201e8c9,44311,1732519720918 started 2024-11-25T07:28:42,174 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,174 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,174 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,174 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.HRegionServer(1482): Serving as 5eb3d201e8c9,44311,1732519720918, RpcServer on 5eb3d201e8c9/172.17.0.2:44311, sessionid=0x1014e06dc560002 2024-11-25T07:28:42,174 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(1482): Serving as 5eb3d201e8c9,35771,1732519720972, RpcServer on 5eb3d201e8c9/172.17.0.2:35771, sessionid=0x1014e06dc560003 2024-11-25T07:28:42,174 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(1482): Serving as 5eb3d201e8c9,33753,1732519720813, RpcServer on 5eb3d201e8c9/172.17.0.2:33753, sessionid=0x1014e06dc560001 2024-11-25T07:28:42,175 DEBUG [RS:0;5eb3d201e8c9:33753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T07:28:42,175 DEBUG [RS:2;5eb3d201e8c9:35771 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T07:28:42,175 DEBUG [RS:1;5eb3d201e8c9:44311 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T07:28:42,175 DEBUG [RS:2;5eb3d201e8c9:35771 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5eb3d201e8c9,35771,1732519720972 2024-11-25T07:28:42,175 DEBUG [RS:1;5eb3d201e8c9:44311 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5eb3d201e8c9,44311,1732519720918 2024-11-25T07:28:42,175 DEBUG [RS:0;5eb3d201e8c9:33753 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5eb3d201e8c9,33753,1732519720813 2024-11-25T07:28:42,176 DEBUG [RS:1;5eb3d201e8c9:44311 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5eb3d201e8c9,44311,1732519720918' 2024-11-25T07:28:42,176 DEBUG [RS:2;5eb3d201e8c9:35771 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5eb3d201e8c9,35771,1732519720972' 2024-11-25T07:28:42,176 DEBUG [RS:0;5eb3d201e8c9:33753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5eb3d201e8c9,33753,1732519720813' 2024-11-25T07:28:42,176 DEBUG [RS:2;5eb3d201e8c9:35771 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T07:28:42,176 DEBUG [RS:1;5eb3d201e8c9:44311 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T07:28:42,176 DEBUG [RS:0;5eb3d201e8c9:33753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T07:28:42,177 DEBUG [RS:1;5eb3d201e8c9:44311 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T07:28:42,177 DEBUG [RS:0;5eb3d201e8c9:33753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T07:28:42,177 DEBUG [RS:2;5eb3d201e8c9:35771 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T07:28:42,177 DEBUG [RS:0;5eb3d201e8c9:33753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T07:28:42,178 DEBUG [RS:1;5eb3d201e8c9:44311 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T07:28:42,178 DEBUG [RS:0;5eb3d201e8c9:33753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T07:28:42,178 DEBUG [RS:1;5eb3d201e8c9:44311 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T07:28:42,178 DEBUG [RS:2;5eb3d201e8c9:35771 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T07:28:42,178 DEBUG [RS:2;5eb3d201e8c9:35771 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T07:28:42,178 DEBUG [RS:0;5eb3d201e8c9:33753 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5eb3d201e8c9,33753,1732519720813 2024-11-25T07:28:42,178 DEBUG [RS:1;5eb3d201e8c9:44311 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5eb3d201e8c9,44311,1732519720918 2024-11-25T07:28:42,178 DEBUG [RS:2;5eb3d201e8c9:35771 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5eb3d201e8c9,35771,1732519720972 2024-11-25T07:28:42,178 DEBUG [RS:1;5eb3d201e8c9:44311 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5eb3d201e8c9,44311,1732519720918' 2024-11-25T07:28:42,178 DEBUG [RS:2;5eb3d201e8c9:35771 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5eb3d201e8c9,35771,1732519720972' 2024-11-25T07:28:42,178 DEBUG [RS:0;5eb3d201e8c9:33753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5eb3d201e8c9,33753,1732519720813' 2024-11-25T07:28:42,178 DEBUG [RS:2;5eb3d201e8c9:35771 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T07:28:42,178 DEBUG [RS:1;5eb3d201e8c9:44311 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T07:28:42,178 DEBUG [RS:0;5eb3d201e8c9:33753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T07:28:42,179 DEBUG [RS:1;5eb3d201e8c9:44311 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T07:28:42,179 DEBUG [RS:2;5eb3d201e8c9:35771 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T07:28:42,179 DEBUG [RS:0;5eb3d201e8c9:33753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T07:28:42,180 DEBUG [RS:2;5eb3d201e8c9:35771 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T07:28:42,180 WARN [5eb3d201e8c9:33903 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:28:42,180 DEBUG [RS:0;5eb3d201e8c9:33753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T07:28:42,180 DEBUG [RS:1;5eb3d201e8c9:44311 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T07:28:42,180 INFO [RS:2;5eb3d201e8c9:35771 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T07:28:42,180 INFO [RS:0;5eb3d201e8c9:33753 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T07:28:42,180 INFO [RS:1;5eb3d201e8c9:44311 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T07:28:42,180 INFO [RS:2;5eb3d201e8c9:35771 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T07:28:42,180 INFO [RS:0;5eb3d201e8c9:33753 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T07:28:42,180 INFO [RS:1;5eb3d201e8c9:44311 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T07:28:42,285 INFO [RS:1;5eb3d201e8c9:44311 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T07:28:42,286 INFO [RS:2;5eb3d201e8c9:35771 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T07:28:42,286 INFO [RS:0;5eb3d201e8c9:33753 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T07:28:42,289 INFO [RS:0;5eb3d201e8c9:33753 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5eb3d201e8c9%2C33753%2C1732519720813, suffix=, logDir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,33753,1732519720813, archiveDir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/oldWALs, maxLogs=32 2024-11-25T07:28:42,289 INFO [RS:1;5eb3d201e8c9:44311 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5eb3d201e8c9%2C44311%2C1732519720918, suffix=, logDir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,44311,1732519720918, archiveDir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/oldWALs, maxLogs=32 2024-11-25T07:28:42,289 INFO [RS:2;5eb3d201e8c9:35771 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5eb3d201e8c9%2C35771%2C1732519720972, suffix=, logDir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972, archiveDir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/oldWALs, maxLogs=32 2024-11-25T07:28:42,309 DEBUG [RS:0;5eb3d201e8c9:33753 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,33753,1732519720813/5eb3d201e8c9%2C33753%2C1732519720813.1732519722293, exclude list is [], retry=0 2024-11-25T07:28:42,314 DEBUG [RS:2;5eb3d201e8c9:35771 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972/5eb3d201e8c9%2C35771%2C1732519720972.1732519722293, exclude list is [], retry=0 2024-11-25T07:28:42,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:42,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:42,314 DEBUG [RS:1;5eb3d201e8c9:44311 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,44311,1732519720918/5eb3d201e8c9%2C44311%2C1732519720918.1732519722293, exclude list is [], retry=0 2024-11-25T07:28:42,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:42,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:42,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:42,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:42,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:42,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:42,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:42,323 INFO [RS:0;5eb3d201e8c9:33753 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,33753,1732519720813/5eb3d201e8c9%2C33753%2C1732519720813.1732519722293 2024-11-25T07:28:42,325 DEBUG [RS:0;5eb3d201e8c9:33753 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:37263:37263)] 2024-11-25T07:28:42,328 INFO [RS:1;5eb3d201e8c9:44311 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,44311,1732519720918/5eb3d201e8c9%2C44311%2C1732519720918.1732519722293 2024-11-25T07:28:42,329 INFO [RS:2;5eb3d201e8c9:35771 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972/5eb3d201e8c9%2C35771%2C1732519720972.1732519722293 2024-11-25T07:28:42,331 DEBUG [RS:1;5eb3d201e8c9:44311 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:37263:37263)] 2024-11-25T07:28:42,331 DEBUG [RS:2;5eb3d201e8c9:35771 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-25T07:28:42,432 DEBUG [5eb3d201e8c9:33903 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-25T07:28:42,440 DEBUG [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(204): Hosts are {5eb3d201e8c9=0} racks are {/default-rack=0} 2024-11-25T07:28:42,447 DEBUG [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T07:28:42,447 DEBUG [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T07:28:42,447 DEBUG [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-25T07:28:42,447 DEBUG [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T07:28:42,447 DEBUG [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T07:28:42,448 DEBUG [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-25T07:28:42,448 INFO [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T07:28:42,448 INFO [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T07:28:42,448 INFO [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-25T07:28:42,448 DEBUG [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T07:28:42,455 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=5eb3d201e8c9,33753,1732519720813 2024-11-25T07:28:42,461 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5eb3d201e8c9,33753,1732519720813, state=OPENING 2024-11-25T07:28:42,466 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T07:28:42,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35771-0x1014e06dc560003, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:42,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:42,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:42,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:28:42,470 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T07:28:42,470 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T07:28:42,470 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T07:28:42,471 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T07:28:42,472 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T07:28:42,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=5eb3d201e8c9,33753,1732519720813}] 2024-11-25T07:28:42,649 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T07:28:42,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60531, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T07:28:42,663 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T07:28:42,664 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:42,664 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T07:28:42,667 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5eb3d201e8c9%2C33753%2C1732519720813.meta, suffix=.meta, logDir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,33753,1732519720813, archiveDir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/oldWALs, maxLogs=32 2024-11-25T07:28:42,683 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,33753,1732519720813/5eb3d201e8c9%2C33753%2C1732519720813.meta.1732519722669.meta, exclude list is [], retry=0 2024-11-25T07:28:42,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:42,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:42,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:42,691 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,33753,1732519720813/5eb3d201e8c9%2C33753%2C1732519720813.meta.1732519722669.meta 2024-11-25T07:28:42,691 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-25T07:28:42,691 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:28:42,693 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T07:28:42,696 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T07:28:42,700 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T07:28:42,704 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T07:28:42,705 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:42,705 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T07:28:42,705 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T07:28:42,709 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T07:28:42,710 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T07:28:42,710 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:42,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:28:42,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T07:28:42,713 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T07:28:42,713 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:42,714 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:28:42,714 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T07:28:42,715 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T07:28:42,715 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:42,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:28:42,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T07:28:42,717 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T07:28:42,717 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:42,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:28:42,718 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T07:28:42,720 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740 2024-11-25T07:28:42,723 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740 2024-11-25T07:28:42,726 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T07:28:42,726 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T07:28:42,727 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T07:28:42,730 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T07:28:42,732 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64739607, jitterRate=-0.035304680466651917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T07:28:42,732 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T07:28:42,734 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732519722706Writing region info on filesystem at 1732519722706Initializing all the Stores at 1732519722708 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732519722708Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732519722709 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519722709Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732519722709Cleaning up temporary data from old regions at 1732519722726 (+17 ms)Running coprocessor post-open hooks at 1732519722732 (+6 ms)Region opened successfully at 1732519722734 (+2 ms) 2024-11-25T07:28:42,743 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732519722639 2024-11-25T07:28:42,757 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T07:28:42,757 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=5eb3d201e8c9,33753,1732519720813 2024-11-25T07:28:42,757 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T07:28:42,760 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5eb3d201e8c9,33753,1732519720813, state=OPEN 2024-11-25T07:28:42,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T07:28:42,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T07:28:42,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T07:28:42,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35771-0x1014e06dc560003, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T07:28:42,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T07:28:42,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T07:28:42,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T07:28:42,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T07:28:42,763 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=5eb3d201e8c9,33753,1732519720813 2024-11-25T07:28:42,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T07:28:42,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5eb3d201e8c9,33753,1732519720813 in 290 msec 2024-11-25T07:28:42,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T07:28:42,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 754 msec 2024-11-25T07:28:42,775 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T07:28:42,775 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T07:28:42,799 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T07:28:42,801 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,33753,1732519720813, seqNum=-1] 2024-11-25T07:28:42,852 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T07:28:42,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34809, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T07:28:42,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0620 sec 2024-11-25T07:28:42,883 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732519722883, completionTime=-1 2024-11-25T07:28:42,885 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-25T07:28:42,886 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T07:28:42,909 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-25T07:28:42,909 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732519782909 2024-11-25T07:28:42,909 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732519842909 2024-11-25T07:28:42,909 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-25T07:28:42,911 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-25T07:28:42,917 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,33903,1732519720118-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,918 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,33903,1732519720118-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,918 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,33903,1732519720118-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,919 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-5eb3d201e8c9:33903, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,920 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,920 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:42,926 DEBUG [master/5eb3d201e8c9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T07:28:42,953 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.897sec 2024-11-25T07:28:42,955 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T07:28:42,957 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T07:28:42,958 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T07:28:42,959 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T07:28:42,959 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T07:28:42,960 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,33903,1732519720118-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T07:28:42,960 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,33903,1732519720118-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T07:28:42,965 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T07:28:42,966 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T07:28:42,966 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,33903,1732519720118-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:43,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20137efd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T07:28:43,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,33903,-1 for getting cluster id 2024-11-25T07:28:43,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T07:28:43,029 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f004f047-221d-470c-b151-81737d8f21a0' 2024-11-25T07:28:43,031 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T07:28:43,031 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f004f047-221d-470c-b151-81737d8f21a0" 2024-11-25T07:28:43,032 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@140f033d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T07:28:43,032 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,33903,-1] 2024-11-25T07:28:43,034 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T07:28:43,036 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:28:43,037 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T07:28:43,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dcb5e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T07:28:43,041 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T07:28:43,047 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,33753,1732519720813, seqNum=-1] 2024-11-25T07:28:43,048 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T07:28:43,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T07:28:43,069 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=5eb3d201e8c9,33903,1732519720118 2024-11-25T07:28:43,070 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:44261/hbase 2024-11-25T07:28:43,085 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=364, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=8709 2024-11-25T07:28:43,105 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:43,109 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:43,109 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T07:28:43,114 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-97903272, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/hregion-97903272, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:43,135 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-97903272/hregion-97903272.1732519723116, exclude list is [], retry=0 2024-11-25T07:28:43,139 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:43,139 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:43,140 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:43,144 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-97903272/hregion-97903272.1732519723116 2024-11-25T07:28:43,145 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-25T07:28:43,145 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 553a3ef804ed200e6a123fba6398b73c, NAME => 'testReplayEditsWrittenIntoWAL,,1732519723106.553a3ef804ed200e6a123fba6398b73c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44261/hbase 2024-11-25T07:28:43,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741838_1014 (size=64) 2024-11-25T07:28:43,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741838_1014 (size=64) 2024-11-25T07:28:43,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741838_1014 (size=64) 2024-11-25T07:28:43,161 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732519723106.553a3ef804ed200e6a123fba6398b73c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:43,163 INFO [StoreOpener-553a3ef804ed200e6a123fba6398b73c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 553a3ef804ed200e6a123fba6398b73c 2024-11-25T07:28:43,166 INFO [StoreOpener-553a3ef804ed200e6a123fba6398b73c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 553a3ef804ed200e6a123fba6398b73c columnFamilyName a 2024-11-25T07:28:43,166 DEBUG [StoreOpener-553a3ef804ed200e6a123fba6398b73c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:43,167 INFO [StoreOpener-553a3ef804ed200e6a123fba6398b73c-1 {}] regionserver.HStore(327): Store=553a3ef804ed200e6a123fba6398b73c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:43,167 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 553a3ef804ed200e6a123fba6398b73c 2024-11-25T07:28:43,168 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c 2024-11-25T07:28:43,168 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c 2024-11-25T07:28:43,169 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 553a3ef804ed200e6a123fba6398b73c 2024-11-25T07:28:43,169 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 553a3ef804ed200e6a123fba6398b73c 2024-11-25T07:28:43,172 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 553a3ef804ed200e6a123fba6398b73c 2024-11-25T07:28:43,175 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:28:43,176 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 553a3ef804ed200e6a123fba6398b73c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66691928, jitterRate=-0.006212830543518066}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:28:43,177 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 553a3ef804ed200e6a123fba6398b73c: Writing region info on filesystem at 1732519723161Initializing all the Stores at 1732519723163 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519723163Cleaning up temporary data from old regions at 1732519723169 (+6 ms)Region opened successfully at 1732519723177 (+8 ms) 2024-11-25T07:28:43,177 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 553a3ef804ed200e6a123fba6398b73c, disabling compactions & flushes 2024-11-25T07:28:43,177 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732519723106.553a3ef804ed200e6a123fba6398b73c. 2024-11-25T07:28:43,177 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732519723106.553a3ef804ed200e6a123fba6398b73c. 2024-11-25T07:28:43,177 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732519723106.553a3ef804ed200e6a123fba6398b73c. after waiting 0 ms 2024-11-25T07:28:43,177 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732519723106.553a3ef804ed200e6a123fba6398b73c. 2024-11-25T07:28:43,178 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732519723106.553a3ef804ed200e6a123fba6398b73c. 2024-11-25T07:28:43,178 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 553a3ef804ed200e6a123fba6398b73c: Waiting for close lock at 1732519723177Disabling compacts and flushes for region at 1732519723177Disabling writes for close at 1732519723177Writing region close event to WAL at 1732519723178 (+1 ms)Closed at 1732519723178 2024-11-25T07:28:43,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741837_1013 (size=93) 2024-11-25T07:28:43,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741837_1013 (size=93) 2024-11-25T07:28:43,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741837_1013 (size=93) 2024-11-25T07:28:43,191 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T07:28:43,191 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-97903272:(num 1732519723116) 2024-11-25T07:28:43,193 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-25T07:28:43,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741839_1015 (size=369) 2024-11-25T07:28:43,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741839_1015 (size=369) 2024-11-25T07:28:43,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741839_1015 (size=369) 2024-11-25T07:28:43,211 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-25T07:28:43,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741840_1016 (size=231) 2024-11-25T07:28:43,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741840_1016 (size=231) 2024-11-25T07:28:43,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741840_1016 (size=231) 2024-11-25T07:28:43,246 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-1, size=369 (369bytes) 2024-11-25T07:28:43,247 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-25T07:28:43,247 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-25T07:28:43,248 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-1 2024-11-25T07:28:43,254 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-1 after 3ms 2024-11-25T07:28:43,260 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-1 took 15ms 2024-11-25T07:28:43,263 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-1 so closing down 2024-11-25T07:28:43,266 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-25T07:28:43,268 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/0000000000000000001-wal-1.temp 2024-11-25T07:28:43,268 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T07:28:43,269 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T07:28:43,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741841_1017 (size=369) 2024-11-25T07:28:43,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741841_1017 (size=369) 2024-11-25T07:28:43,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741841_1017 (size=369) 2024-11-25T07:28:43,280 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-25T07:28:43,283 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/0000000000000000002 2024-11-25T07:28:43,287 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 22 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-25T07:28:43,287 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-1, journal: Splitting hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-1, size=369 (369bytes) at 1732519723246Finishing writing output for hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-1 so closing down at 1732519723263 (+17 ms)Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/0000000000000000001-wal-1.temp at 1732519723268 (+5 ms)3 split writer threads finished at 1732519723269 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732519723280 (+11 ms)Rename recovered edits hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/0000000000000000002 at 1732519723283 (+3 ms)Processed 2 edits across 1 Regions in 22 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1732519723287 (+4 ms) 2024-11-25T07:28:43,303 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-2, size=231 (231bytes) 2024-11-25T07:28:43,303 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-2 2024-11-25T07:28:43,304 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-2 after 1ms 2024-11-25T07:28:43,308 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-2 took 5ms 2024-11-25T07:28:43,310 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-2 so closing down 2024-11-25T07:28:43,310 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T07:28:43,313 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-25T07:28:43,315 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/0000000000000000002-wal-2.temp 2024-11-25T07:28:43,315 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T07:28:43,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741842_1018 (size=231) 2024-11-25T07:28:43,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741842_1018 (size=231) 2024-11-25T07:28:43,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741842_1018 (size=231) 2024-11-25T07:28:43,329 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-25T07:28:43,335 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-11-25T07:28:43,336 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-25T07:28:43,336 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-2, journal: Splitting hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-2, size=231 (231bytes) at 1732519723303Finishing writing output for hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-2 so closing down at 1732519723310 (+7 ms)Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/0000000000000000002-wal-2.temp at 1732519723315 (+5 ms)3 split writer threads finished at 1732519723315Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732519723329 (+14 ms)Processed 1 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1732519723336 (+7 ms) 2024-11-25T07:28:43,337 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:28:43,339 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:43,353 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal.1732519723339, exclude list is [], retry=0 2024-11-25T07:28:43,356 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:43,357 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:43,357 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:43,360 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal.1732519723339 2024-11-25T07:28:43,360 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:43429:43429)] 2024-11-25T07:28:43,360 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 553a3ef804ed200e6a123fba6398b73c, NAME => 'testReplayEditsWrittenIntoWAL,,1732519723106.553a3ef804ed200e6a123fba6398b73c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:28:43,360 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732519723106.553a3ef804ed200e6a123fba6398b73c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:43,360 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 553a3ef804ed200e6a123fba6398b73c 2024-11-25T07:28:43,360 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 553a3ef804ed200e6a123fba6398b73c 2024-11-25T07:28:43,362 INFO [StoreOpener-553a3ef804ed200e6a123fba6398b73c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 553a3ef804ed200e6a123fba6398b73c 2024-11-25T07:28:43,364 INFO [StoreOpener-553a3ef804ed200e6a123fba6398b73c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 553a3ef804ed200e6a123fba6398b73c columnFamilyName a 2024-11-25T07:28:43,364 DEBUG [StoreOpener-553a3ef804ed200e6a123fba6398b73c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:43,365 INFO [StoreOpener-553a3ef804ed200e6a123fba6398b73c-1 {}] regionserver.HStore(327): Store=553a3ef804ed200e6a123fba6398b73c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:43,365 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 553a3ef804ed200e6a123fba6398b73c 2024-11-25T07:28:43,366 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c 2024-11-25T07:28:43,369 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c 2024-11-25T07:28:43,371 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/0000000000000000002 2024-11-25T07:28:43,381 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/0000000000000000002 2024-11-25T07:28:43,386 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 553a3ef804ed200e6a123fba6398b73c 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-25T07:28:43,440 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/.tmp/a/5d0564a5fd96469384bb7f302813aaa1 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732519723191/Put/seqid=0 2024-11-25T07:28:43,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741844_1020 (size=5170) 2024-11-25T07:28:43,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741844_1020 (size=5170) 2024-11-25T07:28:43,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741844_1020 (size=5170) 2024-11-25T07:28:43,455 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/.tmp/a/5d0564a5fd96469384bb7f302813aaa1 2024-11-25T07:28:43,501 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/.tmp/a/5d0564a5fd96469384bb7f302813aaa1 as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/a/5d0564a5fd96469384bb7f302813aaa1 2024-11-25T07:28:43,511 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/a/5d0564a5fd96469384bb7f302813aaa1, entries=2, sequenceid=2, filesize=5.0 K 2024-11-25T07:28:43,517 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 553a3ef804ed200e6a123fba6398b73c in 131ms, sequenceid=2, compaction requested=false; wal=null 2024-11-25T07:28:43,518 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/0000000000000000002 2024-11-25T07:28:43,519 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 553a3ef804ed200e6a123fba6398b73c 2024-11-25T07:28:43,519 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 553a3ef804ed200e6a123fba6398b73c 2024-11-25T07:28:43,522 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 553a3ef804ed200e6a123fba6398b73c 2024-11-25T07:28:43,526 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/553a3ef804ed200e6a123fba6398b73c/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-25T07:28:43,527 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 553a3ef804ed200e6a123fba6398b73c; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59983718, jitterRate=-0.10617294907569885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:28:43,528 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 553a3ef804ed200e6a123fba6398b73c: Writing region info on filesystem at 1732519723361Initializing all the Stores at 1732519723362 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519723362Obtaining lock to block concurrent updates at 1732519723386 (+24 ms)Preparing flush snapshotting stores in 553a3ef804ed200e6a123fba6398b73c at 1732519723386Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732519723106.553a3ef804ed200e6a123fba6398b73c., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732519723390 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732519723106.553a3ef804ed200e6a123fba6398b73c. at 1732519723390Flushing 553a3ef804ed200e6a123fba6398b73c/a: creating writer at 1732519723392 (+2 ms)Flushing 553a3ef804ed200e6a123fba6398b73c/a: appending metadata at 1732519723431 (+39 ms)Flushing 553a3ef804ed200e6a123fba6398b73c/a: closing flushed file at 1732519723434 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d64849a: reopening flushed file at 1732519723499 (+65 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 553a3ef804ed200e6a123fba6398b73c in 131ms, sequenceid=2, compaction requested=false; wal=null at 1732519723517 (+18 ms)Cleaning up temporary data from old regions at 1732519723519 (+2 ms)Region opened successfully at 1732519723528 (+9 ms) 2024-11-25T07:28:43,559 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=376 (was 364) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:55628 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:54878 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-661403191-172.17.0.2-1732519717195:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:55758 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:46306 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:46324 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:44261/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-661403191-172.17.0.2-1732519717195:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-661403191-172.17.0.2-1732519717195:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:54710 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=8690 (was 8709) 2024-11-25T07:28:43,569 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=376, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=8689 2024-11-25T07:28:43,586 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:43,588 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:43,589 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T07:28:43,592 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23064334, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/hregion-23064334, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:43,606 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23064334/hregion-23064334.1732519723593, exclude list is [], retry=0 2024-11-25T07:28:43,610 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:43,610 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:43,610 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:43,613 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23064334/hregion-23064334.1732519723593 2024-11-25T07:28:43,613 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:43429:43429)] 2024-11-25T07:28:43,613 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 928210546234623964b40898deccf004, NAME => 'testReplayEditsWrittenIntoWAL,,1732519723586.928210546234623964b40898deccf004.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44261/hbase 2024-11-25T07:28:43,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741846_1022 (size=64) 2024-11-25T07:28:43,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741846_1022 (size=64) 2024-11-25T07:28:43,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741846_1022 (size=64) 2024-11-25T07:28:43,627 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732519723586.928210546234623964b40898deccf004.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:43,629 INFO [StoreOpener-928210546234623964b40898deccf004-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 928210546234623964b40898deccf004 2024-11-25T07:28:43,632 INFO [StoreOpener-928210546234623964b40898deccf004-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 928210546234623964b40898deccf004 columnFamilyName a 2024-11-25T07:28:43,632 DEBUG [StoreOpener-928210546234623964b40898deccf004-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:43,632 INFO [StoreOpener-928210546234623964b40898deccf004-1 {}] regionserver.HStore(327): Store=928210546234623964b40898deccf004/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:43,633 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 928210546234623964b40898deccf004 2024-11-25T07:28:43,634 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004 2024-11-25T07:28:43,635 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004 2024-11-25T07:28:43,635 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 928210546234623964b40898deccf004 2024-11-25T07:28:43,635 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 928210546234623964b40898deccf004 2024-11-25T07:28:43,637 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 928210546234623964b40898deccf004 2024-11-25T07:28:43,641 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:28:43,641 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 928210546234623964b40898deccf004; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61355904, jitterRate=-0.08572578430175781}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:28:43,642 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 928210546234623964b40898deccf004: Writing region info on filesystem at 1732519723627Initializing all the Stores at 1732519723629 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519723629Cleaning up temporary data from old regions at 1732519723635 (+6 ms)Region opened successfully at 1732519723641 (+6 ms) 2024-11-25T07:28:43,642 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 928210546234623964b40898deccf004, disabling compactions & flushes 2024-11-25T07:28:43,642 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732519723586.928210546234623964b40898deccf004. 2024-11-25T07:28:43,642 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732519723586.928210546234623964b40898deccf004. 2024-11-25T07:28:43,642 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732519723586.928210546234623964b40898deccf004. after waiting 0 ms 2024-11-25T07:28:43,642 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732519723586.928210546234623964b40898deccf004. 2024-11-25T07:28:43,643 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732519723586.928210546234623964b40898deccf004. 2024-11-25T07:28:43,643 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 928210546234623964b40898deccf004: Waiting for close lock at 1732519723642Disabling compacts and flushes for region at 1732519723642Disabling writes for close at 1732519723642Writing region close event to WAL at 1732519723643 (+1 ms)Closed at 1732519723643 2024-11-25T07:28:43,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741845_1021 (size=93) 2024-11-25T07:28:43,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741845_1021 (size=93) 2024-11-25T07:28:43,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741845_1021 (size=93) 2024-11-25T07:28:43,650 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T07:28:43,650 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23064334:(num 1732519723593) 2024-11-25T07:28:43,651 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-25T07:28:43,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741847_1023 (size=369) 2024-11-25T07:28:43,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741847_1023 (size=369) 2024-11-25T07:28:43,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741847_1023 (size=369) 2024-11-25T07:28:43,670 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-25T07:28:43,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741848_1024 (size=231) 2024-11-25T07:28:43,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741848_1024 (size=231) 2024-11-25T07:28:43,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741848_1024 (size=231) 2024-11-25T07:28:43,698 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-2, size=231 (231bytes) 2024-11-25T07:28:43,698 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-2 2024-11-25T07:28:43,699 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-2 after 1ms 2024-11-25T07:28:43,702 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-2 took 4ms 2024-11-25T07:28:43,704 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-2 so closing down 2024-11-25T07:28:43,704 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T07:28:43,707 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-25T07:28:43,708 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000002-wal-2.temp 2024-11-25T07:28:43,708 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T07:28:43,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741849_1025 (size=231) 2024-11-25T07:28:43,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741849_1025 (size=231) 2024-11-25T07:28:43,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741849_1025 (size=231) 2024-11-25T07:28:43,717 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-25T07:28:43,719 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000002 2024-11-25T07:28:43,719 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-25T07:28:43,719 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-2, journal: Splitting hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-2, size=231 (231bytes) at 1732519723698Finishing writing output for hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-2 so closing down at 1732519723704 (+6 ms)Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000002-wal-2.temp at 1732519723708 (+4 ms)3 split writer threads finished at 1732519723708Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732519723717 (+9 ms)Rename recovered edits hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000002 at 1732519723719 (+2 ms)Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1732519723719 2024-11-25T07:28:43,734 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-1, size=369 (369bytes) 2024-11-25T07:28:43,734 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-1 2024-11-25T07:28:43,735 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-1 after 1ms 2024-11-25T07:28:43,738 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-1 took 4ms 2024-11-25T07:28:43,741 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-1 so closing down 2024-11-25T07:28:43,741 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T07:28:43,743 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-25T07:28:43,744 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000001-wal-1.temp 2024-11-25T07:28:43,744 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T07:28:43,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741850_1026 (size=369) 2024-11-25T07:28:43,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741850_1026 (size=369) 2024-11-25T07:28:43,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741850_1026 (size=369) 2024-11-25T07:28:43,753 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-25T07:28:43,758 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000002, length=231 2024-11-25T07:28:43,760 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000002 2024-11-25T07:28:43,760 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-25T07:28:43,761 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-1, journal: Splitting hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-1, size=369 (369bytes) at 1732519723734Finishing writing output for hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-1 so closing down at 1732519723741 (+7 ms)Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000001-wal-1.temp at 1732519723744 (+3 ms)3 split writer threads finished at 1732519723744Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732519723753 (+9 ms)Rename recovered edits hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000002 at 1732519723760 (+7 ms)Processed 2 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1732519723760 2024-11-25T07:28:43,761 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:28:43,763 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:43,777 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal.1732519723764, exclude list is [], retry=0 2024-11-25T07:28:43,781 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:43,781 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:43,782 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:43,784 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal.1732519723764 2024-11-25T07:28:43,784 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:37263:37263)] 2024-11-25T07:28:43,784 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 928210546234623964b40898deccf004, NAME => 'testReplayEditsWrittenIntoWAL,,1732519723586.928210546234623964b40898deccf004.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:28:43,785 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732519723586.928210546234623964b40898deccf004.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:43,785 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 928210546234623964b40898deccf004 2024-11-25T07:28:43,785 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 928210546234623964b40898deccf004 2024-11-25T07:28:43,787 INFO [StoreOpener-928210546234623964b40898deccf004-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 928210546234623964b40898deccf004 2024-11-25T07:28:43,788 INFO [StoreOpener-928210546234623964b40898deccf004-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 928210546234623964b40898deccf004 columnFamilyName a 2024-11-25T07:28:43,789 DEBUG [StoreOpener-928210546234623964b40898deccf004-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:43,789 INFO [StoreOpener-928210546234623964b40898deccf004-1 {}] regionserver.HStore(327): Store=928210546234623964b40898deccf004/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:43,790 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 928210546234623964b40898deccf004 2024-11-25T07:28:43,790 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004 2024-11-25T07:28:43,792 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004 2024-11-25T07:28:43,793 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000002 2024-11-25T07:28:43,798 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000002 2024-11-25T07:28:43,798 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 928210546234623964b40898deccf004 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-25T07:28:43,816 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/.tmp/a/8a06577ec58949c1adb46cbd94c593dd is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732519723650/Put/seqid=0 2024-11-25T07:28:43,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741852_1028 (size=5170) 2024-11-25T07:28:43,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741852_1028 (size=5170) 2024-11-25T07:28:43,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741852_1028 (size=5170) 2024-11-25T07:28:43,826 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/.tmp/a/8a06577ec58949c1adb46cbd94c593dd 2024-11-25T07:28:43,836 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/.tmp/a/8a06577ec58949c1adb46cbd94c593dd as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/a/8a06577ec58949c1adb46cbd94c593dd 2024-11-25T07:28:43,843 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/a/8a06577ec58949c1adb46cbd94c593dd, entries=2, sequenceid=2, filesize=5.0 K 2024-11-25T07:28:43,844 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 928210546234623964b40898deccf004 in 46ms, sequenceid=2, compaction requested=false; wal=null 2024-11-25T07:28:43,845 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/0000000000000000002 2024-11-25T07:28:43,845 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 928210546234623964b40898deccf004 2024-11-25T07:28:43,845 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 928210546234623964b40898deccf004 2024-11-25T07:28:43,848 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 928210546234623964b40898deccf004 2024-11-25T07:28:43,850 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/928210546234623964b40898deccf004/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-25T07:28:43,852 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 928210546234623964b40898deccf004; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65359930, jitterRate=-0.026061147451400757}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:28:43,852 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 928210546234623964b40898deccf004: Writing region info on filesystem at 1732519723785Initializing all the Stores at 1732519723786 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519723786Obtaining lock to block concurrent updates at 1732519723798 (+12 ms)Preparing flush snapshotting stores in 928210546234623964b40898deccf004 at 1732519723798Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732519723586.928210546234623964b40898deccf004., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732519723798Flushing stores of testReplayEditsWrittenIntoWAL,,1732519723586.928210546234623964b40898deccf004. at 1732519723798Flushing 928210546234623964b40898deccf004/a: creating writer at 1732519723798Flushing 928210546234623964b40898deccf004/a: appending metadata at 1732519723814 (+16 ms)Flushing 928210546234623964b40898deccf004/a: closing flushed file at 1732519723814Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70692731: reopening flushed file at 1732519723835 (+21 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 928210546234623964b40898deccf004 in 46ms, sequenceid=2, compaction requested=false; wal=null at 1732519723844 (+9 ms)Cleaning up temporary data from old regions at 1732519723845 (+1 ms)Region opened successfully at 1732519723852 (+7 ms) 2024-11-25T07:28:43,868 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=386 (was 376) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:46396 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:46306 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-661403191-172.17.0.2-1732519717195:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-661403191-172.17.0.2-1732519717195:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:55628 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:55804 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:54958 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-661403191-172.17.0.2-1732519717195:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=8684 (was 8689) 2024-11-25T07:28:43,879 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=386, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=8684 2024-11-25T07:28:43,895 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:43,897 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:43,897 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T07:28:43,901 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-41835801, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/hregion-41835801, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:43,915 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-41835801/hregion-41835801.1732519723901, exclude list is [], retry=0 2024-11-25T07:28:43,919 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:43,919 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:43,919 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:43,922 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-41835801/hregion-41835801.1732519723901 2024-11-25T07:28:43,922 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:43429:43429)] 2024-11-25T07:28:43,922 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ec4b981574679aa525887e8e6eae7851, NAME => 'testReplayEditsWrittenIntoWAL,,1732519723895.ec4b981574679aa525887e8e6eae7851.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44261/hbase 2024-11-25T07:28:43,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741854_1030 (size=64) 2024-11-25T07:28:43,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741854_1030 (size=64) 2024-11-25T07:28:43,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741854_1030 (size=64) 2024-11-25T07:28:43,934 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732519723895.ec4b981574679aa525887e8e6eae7851.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:43,936 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:43,938 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec4b981574679aa525887e8e6eae7851 columnFamilyName a 2024-11-25T07:28:43,938 DEBUG [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:43,938 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] regionserver.HStore(327): Store=ec4b981574679aa525887e8e6eae7851/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:43,939 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:43,940 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec4b981574679aa525887e8e6eae7851 columnFamilyName b 2024-11-25T07:28:43,940 DEBUG [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:43,941 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] regionserver.HStore(327): Store=ec4b981574679aa525887e8e6eae7851/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:43,941 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:43,943 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec4b981574679aa525887e8e6eae7851 columnFamilyName c 2024-11-25T07:28:43,943 DEBUG [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:43,944 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] regionserver.HStore(327): Store=ec4b981574679aa525887e8e6eae7851/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:43,944 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:43,945 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:43,945 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:43,947 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:43,947 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:43,947 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:28:43,949 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:43,952 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:28:43,952 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ec4b981574679aa525887e8e6eae7851; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72404118, jitterRate=0.07890543341636658}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:28:43,953 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ec4b981574679aa525887e8e6eae7851: Writing region info on filesystem at 1732519723934Initializing all the Stores at 1732519723935 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519723935Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519723936 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519723936Cleaning up temporary data from old regions at 1732519723947 (+11 ms)Region opened successfully at 1732519723953 (+6 ms) 2024-11-25T07:28:43,953 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ec4b981574679aa525887e8e6eae7851, disabling compactions & flushes 2024-11-25T07:28:43,953 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732519723895.ec4b981574679aa525887e8e6eae7851. 2024-11-25T07:28:43,953 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732519723895.ec4b981574679aa525887e8e6eae7851. 2024-11-25T07:28:43,953 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732519723895.ec4b981574679aa525887e8e6eae7851. after waiting 0 ms 2024-11-25T07:28:43,953 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732519723895.ec4b981574679aa525887e8e6eae7851. 2024-11-25T07:28:43,954 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732519723895.ec4b981574679aa525887e8e6eae7851. 2024-11-25T07:28:43,954 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ec4b981574679aa525887e8e6eae7851: Waiting for close lock at 1732519723953Disabling compacts and flushes for region at 1732519723953Disabling writes for close at 1732519723953Writing region close event to WAL at 1732519723953Closed at 1732519723953 2024-11-25T07:28:43,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741853_1029 (size=93) 2024-11-25T07:28:43,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741853_1029 (size=93) 2024-11-25T07:28:43,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741853_1029 (size=93) 2024-11-25T07:28:43,960 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T07:28:43,960 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-41835801:(num 1732519723901) 2024-11-25T07:28:43,960 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:28:43,963 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:43,976 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963, exclude list is [], retry=0 2024-11-25T07:28:43,979 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:43,980 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:43,980 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:43,982 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963 2024-11-25T07:28:43,982 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-25T07:28:44,204 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963, size=0 (0bytes) 2024-11-25T07:28:44,204 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963 might be still open, length is 0 2024-11-25T07:28:44,204 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963 2024-11-25T07:28:44,205 WARN [IPC Server handler 4 on default port 44261 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-25T07:28:44,206 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963 after 2ms 2024-11-25T07:28:45,105 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:55844 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37227:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55844 dst: /127.0.0.1:37227 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37227 remote=/127.0.0.1:55844]. Total timeout mills is 60000, 59051 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:28:45,107 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:46430 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37043:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46430 dst: /127.0.0.1:37043 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:28:45,107 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:54990 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33603:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54990 dst: /127.0.0.1:33603 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:28:45,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741855_1032 (size=470675) 2024-11-25T07:28:45,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741855_1032 (size=470675) 2024-11-25T07:28:45,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741855_1032 (size=470675) 2024-11-25T07:28:48,207 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963 after 4003ms 2024-11-25T07:28:48,209 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T07:28:48,213 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963 took 4009ms 2024-11-25T07:28:48,222 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732519723963.temp 2024-11-25T07:28:48,225 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/recovered.edits/0000000000000000001-wal.1732519723963.temp 2024-11-25T07:28:48,277 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T07:28:48,316 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963; continuing. 2024-11-25T07:28:48,317 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963 so closing down 2024-11-25T07:28:48,317 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T07:28:48,317 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T07:28:48,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741856_1033 (size=470683) 2024-11-25T07:28:48,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741856_1033 (size=470683) 2024-11-25T07:28:48,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741856_1033 (size=470683) 2024-11-25T07:28:48,325 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/recovered.edits/0000000000000000001-wal.1732519723963.temp (wrote 3002 edits, skipped 0 edits in 35 ms) 2024-11-25T07:28:48,327 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/recovered.edits/0000000000000000001-wal.1732519723963.temp to hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/recovered.edits/0000000000000003002 2024-11-25T07:28:48,327 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 114 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963, size=0, length=0, corrupted=false, cancelled=false 2024-11-25T07:28:48,327 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963, journal: Splitting hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963, size=0 (0bytes) at 1732519724204Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/recovered.edits/0000000000000000001-wal.1732519723963.temp at 1732519728225 (+4021 ms)Split 1024 edits, skipped 0 edits. at 1732519728278 (+53 ms)Split 2048 edits, skipped 0 edits. at 1732519728301 (+23 ms)Finishing writing output for hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963 so closing down at 1732519728317 (+16 ms)3 split writer threads finished at 1732519728317Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/recovered.edits/0000000000000000001-wal.1732519723963.temp (wrote 3002 edits, skipped 0 edits in 35 ms) at 1732519728325 (+8 ms)Rename recovered edits hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/recovered.edits/0000000000000000001-wal.1732519723963.temp to hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/recovered.edits/0000000000000003002 at 1732519728327 (+2 ms)Processed 3002 edits across 1 Regions in 114 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963, size=0, length=0, corrupted=false, cancelled=false at 1732519728327 2024-11-25T07:28:48,329 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963 to hdfs://localhost:44261/hbase/oldWALs/wal.1732519723963 2024-11-25T07:28:48,330 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/recovered.edits/0000000000000003002 2024-11-25T07:28:48,331 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:28:48,333 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:48,347 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519728333, exclude list is [], retry=0 2024-11-25T07:28:48,350 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:48,351 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:48,351 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:48,353 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519728333 2024-11-25T07:28:48,354 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:37263:37263)] 2024-11-25T07:28:48,354 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732519723895.ec4b981574679aa525887e8e6eae7851.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:48,356 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:48,358 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec4b981574679aa525887e8e6eae7851 columnFamilyName a 2024-11-25T07:28:48,358 DEBUG [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:48,358 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] regionserver.HStore(327): Store=ec4b981574679aa525887e8e6eae7851/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:48,358 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:48,359 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec4b981574679aa525887e8e6eae7851 columnFamilyName b 2024-11-25T07:28:48,359 DEBUG [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:48,360 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] regionserver.HStore(327): Store=ec4b981574679aa525887e8e6eae7851/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:48,360 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:48,361 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec4b981574679aa525887e8e6eae7851 columnFamilyName c 2024-11-25T07:28:48,361 DEBUG [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:48,362 INFO [StoreOpener-ec4b981574679aa525887e8e6eae7851-1 {}] regionserver.HStore(327): Store=ec4b981574679aa525887e8e6eae7851/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:48,362 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:48,363 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:48,365 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:48,366 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/recovered.edits/0000000000000003002 2024-11-25T07:28:48,397 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-25T07:28:48,795 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ec4b981574679aa525887e8e6eae7851 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-25T07:28:48,837 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/a/c48f6715a95744aaa1ced96d1ff768d1 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732519723989/Put/seqid=0 2024-11-25T07:28:48,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741858_1035 (size=50463) 2024-11-25T07:28:48,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741858_1035 (size=50463) 2024-11-25T07:28:48,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741858_1035 (size=50463) 2024-11-25T07:28:48,849 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/a/c48f6715a95744aaa1ced96d1ff768d1 2024-11-25T07:28:48,857 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/a/c48f6715a95744aaa1ced96d1ff768d1 as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/a/c48f6715a95744aaa1ced96d1ff768d1 2024-11-25T07:28:48,864 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/a/c48f6715a95744aaa1ced96d1ff768d1, entries=754, sequenceid=754, filesize=49.3 K 2024-11-25T07:28:48,864 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for ec4b981574679aa525887e8e6eae7851 in 69ms, sequenceid=754, compaction requested=false; wal=null 2024-11-25T07:28:48,877 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-25T07:28:48,878 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ec4b981574679aa525887e8e6eae7851 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-25T07:28:48,887 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/a/8dfc8679f6d542e08f8283afa425e42c is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732519724029/Put/seqid=0 2024-11-25T07:28:48,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741859_1036 (size=20072) 2024-11-25T07:28:48,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741859_1036 (size=20072) 2024-11-25T07:28:48,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741859_1036 (size=20072) 2024-11-25T07:28:48,904 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/a/8dfc8679f6d542e08f8283afa425e42c 2024-11-25T07:28:48,931 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/b/24d8f121e5e045a89eab1e72c6679158 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732519724053/Put/seqid=0 2024-11-25T07:28:48,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741860_1037 (size=35835) 2024-11-25T07:28:48,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741860_1037 (size=35835) 2024-11-25T07:28:48,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741860_1037 (size=35835) 2024-11-25T07:28:48,942 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/b/24d8f121e5e045a89eab1e72c6679158 2024-11-25T07:28:48,949 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/a/8dfc8679f6d542e08f8283afa425e42c as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/a/8dfc8679f6d542e08f8283afa425e42c 2024-11-25T07:28:48,957 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/a/8dfc8679f6d542e08f8283afa425e42c, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-25T07:28:48,958 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/b/24d8f121e5e045a89eab1e72c6679158 as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/b/24d8f121e5e045a89eab1e72c6679158 2024-11-25T07:28:48,966 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/b/24d8f121e5e045a89eab1e72c6679158, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-25T07:28:48,966 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for ec4b981574679aa525887e8e6eae7851 in 89ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-25T07:28:48,979 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-25T07:28:48,980 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ec4b981574679aa525887e8e6eae7851 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-25T07:28:48,986 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/b/1f6c33f2beea456c9e19f805db0dafb6 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732519724072/Put/seqid=0 2024-11-25T07:28:48,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741861_1038 (size=35082) 2024-11-25T07:28:48,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741861_1038 (size=35082) 2024-11-25T07:28:48,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741861_1038 (size=35082) 2024-11-25T07:28:49,000 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/b/1f6c33f2beea456c9e19f805db0dafb6 2024-11-25T07:28:49,028 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/c/65bc1df580a0483ab2feee4c4ef35fbf is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732519724100/Put/seqid=0 2024-11-25T07:28:49,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741862_1039 (size=20825) 2024-11-25T07:28:49,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741862_1039 (size=20825) 2024-11-25T07:28:49,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741862_1039 (size=20825) 2024-11-25T07:28:49,042 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/c/65bc1df580a0483ab2feee4c4ef35fbf 2024-11-25T07:28:49,051 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/b/1f6c33f2beea456c9e19f805db0dafb6 as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/b/1f6c33f2beea456c9e19f805db0dafb6 2024-11-25T07:28:49,058 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/b/1f6c33f2beea456c9e19f805db0dafb6, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-25T07:28:49,060 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/c/65bc1df580a0483ab2feee4c4ef35fbf as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/c/65bc1df580a0483ab2feee4c4ef35fbf 2024-11-25T07:28:49,068 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/c/65bc1df580a0483ab2feee4c4ef35fbf, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-25T07:28:49,068 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for ec4b981574679aa525887e8e6eae7851 in 88ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-25T07:28:49,079 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732519724154/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732519723895.ec4b981574679aa525887e8e6eae7851. 2024-11-25T07:28:49,084 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/recovered.edits/0000000000000003002 2024-11-25T07:28:49,085 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-25T07:28:49,085 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ec4b981574679aa525887e8e6eae7851 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-25T07:28:49,097 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/c/3688f532ee534ef380b6e4f46314296a is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732519724109/Put/seqid=0 2024-11-25T07:28:49,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741863_1040 (size=50301) 2024-11-25T07:28:49,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741863_1040 (size=50301) 2024-11-25T07:28:49,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741863_1040 (size=50301) 2024-11-25T07:28:49,111 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/c/3688f532ee534ef380b6e4f46314296a 2024-11-25T07:28:49,122 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3688f532ee534ef380b6e4f46314296a 2024-11-25T07:28:49,125 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/.tmp/c/3688f532ee534ef380b6e4f46314296a as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/c/3688f532ee534ef380b6e4f46314296a 2024-11-25T07:28:49,135 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3688f532ee534ef380b6e4f46314296a 2024-11-25T07:28:49,135 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/c/3688f532ee534ef380b6e4f46314296a, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-25T07:28:49,135 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for ec4b981574679aa525887e8e6eae7851 in 50ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-25T07:28:49,136 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/recovered.edits/0000000000000003002 2024-11-25T07:28:49,138 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:49,138 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:49,138 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T07:28:49,140 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ec4b981574679aa525887e8e6eae7851 2024-11-25T07:28:49,143 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4b981574679aa525887e8e6eae7851/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-25T07:28:49,144 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ec4b981574679aa525887e8e6eae7851; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74667468, jitterRate=0.11263197660446167}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T07:28:49,145 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ec4b981574679aa525887e8e6eae7851: Writing region info on filesystem at 1732519728354Initializing all the Stores at 1732519728356 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519728356Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519728356Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519728356Cleaning up temporary data from old regions at 1732519729138 (+782 ms)Region opened successfully at 1732519729145 (+7 ms) 2024-11-25T07:28:49,213 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ec4b981574679aa525887e8e6eae7851, disabling compactions & flushes 2024-11-25T07:28:49,213 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732519723895.ec4b981574679aa525887e8e6eae7851. 2024-11-25T07:28:49,213 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732519723895.ec4b981574679aa525887e8e6eae7851. 2024-11-25T07:28:49,214 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732519723895.ec4b981574679aa525887e8e6eae7851. after waiting 0 ms 2024-11-25T07:28:49,214 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732519723895.ec4b981574679aa525887e8e6eae7851. 2024-11-25T07:28:49,216 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732519723895.ec4b981574679aa525887e8e6eae7851. 2024-11-25T07:28:49,216 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ec4b981574679aa525887e8e6eae7851: Waiting for close lock at 1732519729213Disabling compacts and flushes for region at 1732519729213Disabling writes for close at 1732519729214 (+1 ms)Writing region close event to WAL at 1732519729216 (+2 ms)Closed at 1732519729216 2024-11-25T07:28:49,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741857_1034 (size=93) 2024-11-25T07:28:49,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741857_1034 (size=93) 2024-11-25T07:28:49,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741857_1034 (size=93) 2024-11-25T07:28:49,224 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T07:28:49,224 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732519728333) 2024-11-25T07:28:49,242 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=402 (was 386) Potentially hanging thread: IPC Client (473829079) connection to localhost/127.0.0.1:37307 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_183105470_22 at /127.0.0.1:55058 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (473829079) connection to localhost/127.0.0.1:41097 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:44261 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37307 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_183105470_22 at /127.0.0.1:46456 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41097 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (473829079) connection to localhost/127.0.0.1:44261 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_183105470_22 at /127.0.0.1:55876 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_183105470_22 at /127.0.0.1:55886 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=116 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=8573 (was 8684) 2024-11-25T07:28:49,258 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=402, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=116, ProcessCount=11, AvailableMemoryMB=8573 2024-11-25T07:28:49,278 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:49,281 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:49,281 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T07:28:49,285 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-02058024, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/hregion-02058024, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:49,301 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-02058024/hregion-02058024.1732519729285, exclude list is [], retry=0 2024-11-25T07:28:49,305 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:49,305 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:49,309 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:49,314 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-02058024/hregion-02058024.1732519729285 2024-11-25T07:28:49,315 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:43429:43429)] 2024-11-25T07:28:49,315 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8fea23943e0cb99a4c6133479c9db18a, NAME => 'test2727,,1732519729279.8fea23943e0cb99a4c6133479c9db18a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44261/hbase 2024-11-25T07:28:49,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741865_1042 (size=43) 2024-11-25T07:28:49,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741865_1042 (size=43) 2024-11-25T07:28:49,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741865_1042 (size=43) 2024-11-25T07:28:49,331 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732519729279.8fea23943e0cb99a4c6133479c9db18a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:49,333 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:49,334 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8fea23943e0cb99a4c6133479c9db18a columnFamilyName a 2024-11-25T07:28:49,335 DEBUG [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:49,335 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] regionserver.HStore(327): Store=8fea23943e0cb99a4c6133479c9db18a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:49,335 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:49,337 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8fea23943e0cb99a4c6133479c9db18a columnFamilyName b 2024-11-25T07:28:49,337 DEBUG [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:49,338 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] regionserver.HStore(327): Store=8fea23943e0cb99a4c6133479c9db18a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:49,338 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:49,339 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8fea23943e0cb99a4c6133479c9db18a columnFamilyName c 2024-11-25T07:28:49,340 DEBUG [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:49,340 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] regionserver.HStore(327): Store=8fea23943e0cb99a4c6133479c9db18a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:49,340 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:49,341 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:49,342 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:49,343 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:49,343 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:49,344 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:28:49,345 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:49,348 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:28:49,349 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8fea23943e0cb99a4c6133479c9db18a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70835198, jitterRate=0.05552670359611511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:28:49,350 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8fea23943e0cb99a4c6133479c9db18a: Writing region info on filesystem at 1732519729331Initializing all the Stores at 1732519729332 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519729332Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519729333 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519729333Cleaning up temporary data from old regions at 1732519729343 (+10 ms)Region opened successfully at 1732519729350 (+7 ms) 2024-11-25T07:28:49,351 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8fea23943e0cb99a4c6133479c9db18a, disabling compactions & flushes 2024-11-25T07:28:49,351 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732519729279.8fea23943e0cb99a4c6133479c9db18a. 2024-11-25T07:28:49,351 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732519729279.8fea23943e0cb99a4c6133479c9db18a. 2024-11-25T07:28:49,351 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732519729279.8fea23943e0cb99a4c6133479c9db18a. after waiting 0 ms 2024-11-25T07:28:49,351 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732519729279.8fea23943e0cb99a4c6133479c9db18a. 2024-11-25T07:28:49,355 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732519729279.8fea23943e0cb99a4c6133479c9db18a. 2024-11-25T07:28:49,355 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8fea23943e0cb99a4c6133479c9db18a: Waiting for close lock at 1732519729351Disabling compacts and flushes for region at 1732519729351Disabling writes for close at 1732519729351Writing region close event to WAL at 1732519729355 (+4 ms)Closed at 1732519729355 2024-11-25T07:28:49,361 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-02058024/hregion-02058024.1732519729285 not finished, retry = 0 2024-11-25T07:28:49,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741864_1041 (size=93) 2024-11-25T07:28:49,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741864_1041 (size=93) 2024-11-25T07:28:49,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741864_1041 (size=93) 2024-11-25T07:28:49,465 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T07:28:49,465 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-02058024:(num 1732519729285) 2024-11-25T07:28:49,465 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:28:49,468 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:49,482 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729468, exclude list is [], retry=0 2024-11-25T07:28:49,485 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:49,486 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:49,486 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:49,489 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729468 2024-11-25T07:28:49,489 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:43429:43429)] 2024-11-25T07:28:49,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741866_1043 (size=344306) 2024-11-25T07:28:49,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741866_1043 (size=344306) 2024-11-25T07:28:49,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741866_1043 (size=344306) 2024-11-25T07:28:49,688 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729468, size=336.2 K (344306bytes) 2024-11-25T07:28:49,689 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729468 2024-11-25T07:28:49,689 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729468 after 0ms 2024-11-25T07:28:49,693 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729468 took 5ms 2024-11-25T07:28:49,700 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732519729468.temp 2024-11-25T07:28:49,701 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000000001-wal.1732519729468.temp 2024-11-25T07:28:49,732 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729468 so closing down 2024-11-25T07:28:49,732 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T07:28:49,732 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T07:28:49,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741867_1044 (size=344306) 2024-11-25T07:28:49,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741867_1044 (size=344306) 2024-11-25T07:28:49,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741867_1044 (size=344306) 2024-11-25T07:28:49,741 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000000001-wal.1732519729468.temp (wrote 3000 edits, skipped 0 edits in 12 ms) 2024-11-25T07:28:49,743 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000000001-wal.1732519729468.temp to hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000003000 2024-11-25T07:28:49,743 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 49 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729468, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-11-25T07:28:49,743 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729468, journal: Splitting hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729468, size=336.2 K (344306bytes) at 1732519729688Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000000001-wal.1732519729468.temp at 1732519729701 (+13 ms)Split 1024 edits, skipped 0 edits. at 1732519729708 (+7 ms)Split 2048 edits, skipped 0 edits. at 1732519729720 (+12 ms)Finishing writing output for hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729468 so closing down at 1732519729732 (+12 ms)3 split writer threads finished at 1732519729732Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000000001-wal.1732519729468.temp (wrote 3000 edits, skipped 0 edits in 12 ms) at 1732519729741 (+9 ms)Rename recovered edits hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000000001-wal.1732519729468.temp to hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000003000 at 1732519729743 (+2 ms)Processed 3000 edits across 1 Regions in 49 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729468, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1732519729743 2024-11-25T07:28:49,746 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729468 to hdfs://localhost:44261/hbase/oldWALs/wal.1732519729468 2024-11-25T07:28:49,747 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000003000 2024-11-25T07:28:49,747 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:28:49,750 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:49,773 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729751, exclude list is [], retry=0 2024-11-25T07:28:49,777 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:49,778 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:49,778 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:49,784 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729751 2024-11-25T07:28:49,784 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-25T07:28:49,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741868_1045 (size=344433) 2024-11-25T07:28:49,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741868_1045 (size=344433) 2024-11-25T07:28:49,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741868_1045 (size=344433) 2024-11-25T07:28:49,983 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729751, size=336.4 K (344433bytes) 2024-11-25T07:28:49,983 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729751 2024-11-25T07:28:49,984 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729751 after 0ms 2024-11-25T07:28:49,991 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729751 took 8ms 2024-11-25T07:28:50,011 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732519729751.temp 2024-11-25T07:28:50,028 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000003001-wal.1732519729751.temp 2024-11-25T07:28:50,045 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729751 so closing down 2024-11-25T07:28:50,045 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T07:28:50,046 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T07:28:50,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741869_1046 (size=344433) 2024-11-25T07:28:50,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741869_1046 (size=344433) 2024-11-25T07:28:50,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741869_1046 (size=344433) 2024-11-25T07:28:50,062 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000003001-wal.1732519729751.temp (wrote 3000 edits, skipped 0 edits in 11 ms) 2024-11-25T07:28:50,063 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000003001-wal.1732519729751.temp to hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000006000 2024-11-25T07:28:50,064 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 60 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729751, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-11-25T07:28:50,064 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729751, journal: Splitting hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729751, size=336.4 K (344433bytes) at 1732519729983Split 1024 edits, skipped 0 edits. at 1732519730021 (+38 ms)Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000003001-wal.1732519729751.temp at 1732519730029 (+8 ms)Split 2048 edits, skipped 0 edits. at 1732519730034 (+5 ms)Finishing writing output for hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729751 so closing down at 1732519730045 (+11 ms)3 split writer threads finished at 1732519730046 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000003001-wal.1732519729751.temp (wrote 3000 edits, skipped 0 edits in 11 ms) at 1732519730062 (+16 ms)Rename recovered edits hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000003001-wal.1732519729751.temp to hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000006000 at 1732519730063 (+1 ms)Processed 3000 edits across 1 Regions in 60 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729751, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1732519730064 (+1 ms) 2024-11-25T07:28:50,066 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519729751 to hdfs://localhost:44261/hbase/oldWALs/wal.1732519729751 2024-11-25T07:28:50,067 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000006000 2024-11-25T07:28:50,067 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:28:50,070 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/test2727-manual,16010,1732519729277, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:50,091 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519730071, exclude list is [], retry=0 2024-11-25T07:28:50,094 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:50,095 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:50,095 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:50,103 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519730071 2024-11-25T07:28:50,103 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-25T07:28:50,103 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8fea23943e0cb99a4c6133479c9db18a, NAME => 'test2727,,1732519729279.8fea23943e0cb99a4c6133479c9db18a.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:28:50,104 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732519729279.8fea23943e0cb99a4c6133479c9db18a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:50,104 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:50,104 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:50,106 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:50,107 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8fea23943e0cb99a4c6133479c9db18a columnFamilyName a 2024-11-25T07:28:50,107 DEBUG [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:50,108 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] regionserver.HStore(327): Store=8fea23943e0cb99a4c6133479c9db18a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:50,108 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:50,109 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8fea23943e0cb99a4c6133479c9db18a columnFamilyName b 2024-11-25T07:28:50,109 DEBUG [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:50,110 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] regionserver.HStore(327): Store=8fea23943e0cb99a4c6133479c9db18a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:50,110 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:50,111 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8fea23943e0cb99a4c6133479c9db18a columnFamilyName c 2024-11-25T07:28:50,111 DEBUG [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:50,112 INFO [StoreOpener-8fea23943e0cb99a4c6133479c9db18a-1 {}] regionserver.HStore(327): Store=8fea23943e0cb99a4c6133479c9db18a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:50,112 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:50,113 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:50,115 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:50,116 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000003000 2024-11-25T07:28:50,191 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000003000 2024-11-25T07:28:50,193 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000006000 2024-11-25T07:28:50,234 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000006000 2024-11-25T07:28:50,235 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8fea23943e0cb99a4c6133479c9db18a 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-25T07:28:50,272 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/.tmp/a/539e31ba576048098b36618642d258a0 is 41, key is test2727/a:100/1732519729791/Put/seqid=0 2024-11-25T07:28:50,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741871_1048 (size=84227) 2024-11-25T07:28:50,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741871_1048 (size=84227) 2024-11-25T07:28:50,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741871_1048 (size=84227) 2024-11-25T07:28:50,293 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/.tmp/a/539e31ba576048098b36618642d258a0 2024-11-25T07:28:50,327 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/.tmp/b/e7851a29b0c9436a934b82fc89df1ccc is 41, key is test2727/b:100/1732519729861/Put/seqid=0 2024-11-25T07:28:50,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741872_1049 (size=84609) 2024-11-25T07:28:50,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741872_1049 (size=84609) 2024-11-25T07:28:50,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741872_1049 (size=84609) 2024-11-25T07:28:50,353 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/.tmp/b/e7851a29b0c9436a934b82fc89df1ccc 2024-11-25T07:28:50,383 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/.tmp/c/53e74dad4e5641fd928f4a8ecc15d014 is 41, key is test2727/c:100/1732519729918/Put/seqid=0 2024-11-25T07:28:50,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741873_1050 (size=84609) 2024-11-25T07:28:50,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741873_1050 (size=84609) 2024-11-25T07:28:50,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741873_1050 (size=84609) 2024-11-25T07:28:50,399 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/.tmp/c/53e74dad4e5641fd928f4a8ecc15d014 2024-11-25T07:28:50,407 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/.tmp/a/539e31ba576048098b36618642d258a0 as hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/a/539e31ba576048098b36618642d258a0 2024-11-25T07:28:50,414 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/a/539e31ba576048098b36618642d258a0, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-25T07:28:50,415 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/.tmp/b/e7851a29b0c9436a934b82fc89df1ccc as hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/b/e7851a29b0c9436a934b82fc89df1ccc 2024-11-25T07:28:50,422 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/b/e7851a29b0c9436a934b82fc89df1ccc, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-25T07:28:50,425 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/.tmp/c/53e74dad4e5641fd928f4a8ecc15d014 as hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/c/53e74dad4e5641fd928f4a8ecc15d014 2024-11-25T07:28:50,435 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/c/53e74dad4e5641fd928f4a8ecc15d014, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-25T07:28:50,436 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 8fea23943e0cb99a4c6133479c9db18a in 202ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-25T07:28:50,437 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000003000 2024-11-25T07:28:50,438 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/0000000000000006000 2024-11-25T07:28:50,439 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:50,439 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:50,445 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:28:50,447 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8fea23943e0cb99a4c6133479c9db18a 2024-11-25T07:28:50,453 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/test2727/8fea23943e0cb99a4c6133479c9db18a/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-25T07:28:50,454 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8fea23943e0cb99a4c6133479c9db18a; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64699739, jitterRate=-0.035898759961128235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:28:50,455 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8fea23943e0cb99a4c6133479c9db18a: Writing region info on filesystem at 1732519730104Initializing all the Stores at 1732519730105 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519730105Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519730106 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519730106Obtaining lock to block concurrent updates at 1732519730235 (+129 ms)Preparing flush snapshotting stores in 8fea23943e0cb99a4c6133479c9db18a at 1732519730235Finished memstore snapshotting test2727,,1732519729279.8fea23943e0cb99a4c6133479c9db18a., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732519730235Flushing stores of test2727,,1732519729279.8fea23943e0cb99a4c6133479c9db18a. at 1732519730235Flushing 8fea23943e0cb99a4c6133479c9db18a/a: creating writer at 1732519730235Flushing 8fea23943e0cb99a4c6133479c9db18a/a: appending metadata at 1732519730271 (+36 ms)Flushing 8fea23943e0cb99a4c6133479c9db18a/a: closing flushed file at 1732519730271Flushing 8fea23943e0cb99a4c6133479c9db18a/b: creating writer at 1732519730301 (+30 ms)Flushing 8fea23943e0cb99a4c6133479c9db18a/b: appending metadata at 1732519730325 (+24 ms)Flushing 8fea23943e0cb99a4c6133479c9db18a/b: closing flushed file at 1732519730325Flushing 8fea23943e0cb99a4c6133479c9db18a/c: creating writer at 1732519730361 (+36 ms)Flushing 8fea23943e0cb99a4c6133479c9db18a/c: appending metadata at 1732519730383 (+22 ms)Flushing 8fea23943e0cb99a4c6133479c9db18a/c: closing flushed file at 1732519730383Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4dc43168: reopening flushed file at 1732519730406 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@286be190: reopening flushed file at 1732519730414 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68af90b2: reopening flushed file at 1732519730422 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 8fea23943e0cb99a4c6133479c9db18a in 202ms, sequenceid=6000, compaction requested=false; wal=null at 1732519730436 (+14 ms)Cleaning up temporary data from old regions at 1732519730439 (+3 ms)Region opened successfully at 1732519730455 (+16 ms) 2024-11-25T07:28:50,457 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-25T07:28:50,457 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8fea23943e0cb99a4c6133479c9db18a, disabling compactions & flushes 2024-11-25T07:28:50,457 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732519729279.8fea23943e0cb99a4c6133479c9db18a. 2024-11-25T07:28:50,458 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732519729279.8fea23943e0cb99a4c6133479c9db18a. 2024-11-25T07:28:50,458 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732519729279.8fea23943e0cb99a4c6133479c9db18a. after waiting 0 ms 2024-11-25T07:28:50,458 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732519729279.8fea23943e0cb99a4c6133479c9db18a. 2024-11-25T07:28:50,459 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732519729279.8fea23943e0cb99a4c6133479c9db18a. 2024-11-25T07:28:50,459 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8fea23943e0cb99a4c6133479c9db18a: Waiting for close lock at 1732519730457Disabling compacts and flushes for region at 1732519730457Disabling writes for close at 1732519730458 (+1 ms)Writing region close event to WAL at 1732519730459 (+1 ms)Closed at 1732519730459 2024-11-25T07:28:50,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741870_1047 (size=93) 2024-11-25T07:28:50,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741870_1047 (size=93) 2024-11-25T07:28:50,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741870_1047 (size=93) 2024-11-25T07:28:50,468 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1732519729277/wal.1732519730071 not finished, retry = 0 2024-11-25T07:28:50,530 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T07:28:50,530 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T07:28:50,533 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T07:28:50,533 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T07:28:50,534 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T07:28:50,534 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T07:28:50,534 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-25T07:28:50,534 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-25T07:28:50,572 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T07:28:50,573 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732519730071) 2024-11-25T07:28:50,590 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=404 (was 402) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:55178 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:46596 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:55876 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 116) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8356 (was 8573) 2024-11-25T07:28:50,603 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=404, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=8354 2024-11-25T07:28:50,619 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:50,626 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732519730618, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:50,627 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732519730626 2024-11-25T07:28:50,634 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732519730618/wal.1732519730626 2024-11-25T07:28:50,636 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-25T07:28:50,638 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 21c6ba20c98650f0f9fb6c03f4b650d9, NAME => 'testSequentialEditLogSeqNum,,1732519730620.21c6ba20c98650f0f9fb6c03f4b650d9.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:28:50,638 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732519730620.21c6ba20c98650f0f9fb6c03f4b650d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:50,638 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 21c6ba20c98650f0f9fb6c03f4b650d9 2024-11-25T07:28:50,638 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 21c6ba20c98650f0f9fb6c03f4b650d9 2024-11-25T07:28:50,639 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:44261/hbase/data/default/testSequentialEditLogSeqNum/21c6ba20c98650f0f9fb6c03f4b650d9 doesn't exist for region: 21c6ba20c98650f0f9fb6c03f4b650d9 on table testSequentialEditLogSeqNum 2024-11-25T07:28:50,640 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 21c6ba20c98650f0f9fb6c03f4b650d9 on table testSequentialEditLogSeqNum 2024-11-25T07:28:50,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741875_1052 (size=62) 2024-11-25T07:28:50,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741875_1052 (size=62) 2024-11-25T07:28:50,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741875_1052 (size=62) 2024-11-25T07:28:50,652 INFO [StoreOpener-21c6ba20c98650f0f9fb6c03f4b650d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 21c6ba20c98650f0f9fb6c03f4b650d9 2024-11-25T07:28:50,653 INFO [StoreOpener-21c6ba20c98650f0f9fb6c03f4b650d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21c6ba20c98650f0f9fb6c03f4b650d9 columnFamilyName a 2024-11-25T07:28:50,653 DEBUG [StoreOpener-21c6ba20c98650f0f9fb6c03f4b650d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:50,654 INFO [StoreOpener-21c6ba20c98650f0f9fb6c03f4b650d9-1 {}] regionserver.HStore(327): Store=21c6ba20c98650f0f9fb6c03f4b650d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:50,654 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 21c6ba20c98650f0f9fb6c03f4b650d9 2024-11-25T07:28:50,655 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testSequentialEditLogSeqNum/21c6ba20c98650f0f9fb6c03f4b650d9 2024-11-25T07:28:50,655 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testSequentialEditLogSeqNum/21c6ba20c98650f0f9fb6c03f4b650d9 2024-11-25T07:28:50,655 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 21c6ba20c98650f0f9fb6c03f4b650d9 2024-11-25T07:28:50,656 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 21c6ba20c98650f0f9fb6c03f4b650d9 2024-11-25T07:28:50,657 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 21c6ba20c98650f0f9fb6c03f4b650d9 2024-11-25T07:28:50,659 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testSequentialEditLogSeqNum/21c6ba20c98650f0f9fb6c03f4b650d9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:28:50,660 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 21c6ba20c98650f0f9fb6c03f4b650d9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70406268, jitterRate=0.04913514852523804}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:28:50,661 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 21c6ba20c98650f0f9fb6c03f4b650d9: Writing region info on filesystem at 1732519730639Initializing all the Stores at 1732519730651 (+12 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519730651Cleaning up temporary data from old regions at 1732519730656 (+5 ms)Region opened successfully at 1732519730661 (+5 ms) 2024-11-25T07:28:50,675 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 21c6ba20c98650f0f9fb6c03f4b650d9 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-25T07:28:50,705 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testSequentialEditLogSeqNum/21c6ba20c98650f0f9fb6c03f4b650d9/.tmp/a/0de5e0a5ac4f4ca2af7571bbdf283a6c is 81, key is testSequentialEditLogSeqNum/a:x0/1732519730661/Put/seqid=0 2024-11-25T07:28:50,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741876_1053 (size=5833) 2024-11-25T07:28:50,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741876_1053 (size=5833) 2024-11-25T07:28:50,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741876_1053 (size=5833) 2024-11-25T07:28:50,717 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testSequentialEditLogSeqNum/21c6ba20c98650f0f9fb6c03f4b650d9/.tmp/a/0de5e0a5ac4f4ca2af7571bbdf283a6c 2024-11-25T07:28:50,727 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testSequentialEditLogSeqNum/21c6ba20c98650f0f9fb6c03f4b650d9/.tmp/a/0de5e0a5ac4f4ca2af7571bbdf283a6c as hdfs://localhost:44261/hbase/data/default/testSequentialEditLogSeqNum/21c6ba20c98650f0f9fb6c03f4b650d9/a/0de5e0a5ac4f4ca2af7571bbdf283a6c 2024-11-25T07:28:50,733 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testSequentialEditLogSeqNum/21c6ba20c98650f0f9fb6c03f4b650d9/a/0de5e0a5ac4f4ca2af7571bbdf283a6c, entries=10, sequenceid=13, filesize=5.7 K 2024-11-25T07:28:50,735 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 21c6ba20c98650f0f9fb6c03f4b650d9 in 60ms, sequenceid=13, compaction requested=false 2024-11-25T07:28:50,735 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 21c6ba20c98650f0f9fb6c03f4b650d9: 2024-11-25T07:28:50,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T07:28:50,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T07:28:50,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T07:28:50,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T07:28:50,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T07:28:50,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741874_1051 (size=3017) 2024-11-25T07:28:50,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741874_1051 (size=3017) 2024-11-25T07:28:50,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741874_1051 (size=3017) 2024-11-25T07:28:50,771 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44261/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732519730618/wal.1732519730626, size=2.9 K (3017bytes) 2024-11-25T07:28:50,771 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44261/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732519730618/wal.1732519730626 2024-11-25T07:28:50,772 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44261/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732519730618/wal.1732519730626 after 1ms 2024-11-25T07:28:50,775 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44261/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732519730618/wal.1732519730626 took 4ms 2024-11-25T07:28:50,778 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44261/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732519730618/wal.1732519730626 so closing down 2024-11-25T07:28:50,778 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T07:28:50,779 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732519730626.temp 2024-11-25T07:28:50,781 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testSequentialEditLogSeqNum/21c6ba20c98650f0f9fb6c03f4b650d9/recovered.edits/0000000000000000003-wal.1732519730626.temp 2024-11-25T07:28:50,781 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T07:28:50,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741877_1054 (size=2433) 2024-11-25T07:28:50,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741877_1054 (size=2433) 2024-11-25T07:28:50,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741877_1054 (size=2433) 2024-11-25T07:28:50,793 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testSequentialEditLogSeqNum/21c6ba20c98650f0f9fb6c03f4b650d9/recovered.edits/0000000000000000003-wal.1732519730626.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-25T07:28:50,795 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44261/hbase/data/default/testSequentialEditLogSeqNum/21c6ba20c98650f0f9fb6c03f4b650d9/recovered.edits/0000000000000000003-wal.1732519730626.temp to hdfs://localhost:44261/hbase/data/default/testSequentialEditLogSeqNum/21c6ba20c98650f0f9fb6c03f4b650d9/recovered.edits/0000000000000000020 2024-11-25T07:28:50,795 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 19 ms; skipped=2; WAL=hdfs://localhost:44261/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732519730618/wal.1732519730626, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-11-25T07:28:50,795 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44261/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732519730618/wal.1732519730626, journal: Splitting hdfs://localhost:44261/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732519730618/wal.1732519730626, size=2.9 K (3017bytes) at 1732519730771Finishing writing output for hdfs://localhost:44261/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732519730618/wal.1732519730626 so closing down at 1732519730778 (+7 ms)Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testSequentialEditLogSeqNum/21c6ba20c98650f0f9fb6c03f4b650d9/recovered.edits/0000000000000000003-wal.1732519730626.temp at 1732519730781 (+3 ms)3 split writer threads finished at 1732519730781Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testSequentialEditLogSeqNum/21c6ba20c98650f0f9fb6c03f4b650d9/recovered.edits/0000000000000000003-wal.1732519730626.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732519730793 (+12 ms)Rename recovered edits hdfs://localhost:44261/hbase/data/default/testSequentialEditLogSeqNum/21c6ba20c98650f0f9fb6c03f4b650d9/recovered.edits/0000000000000000003-wal.1732519730626.temp to hdfs://localhost:44261/hbase/data/default/testSequentialEditLogSeqNum/21c6ba20c98650f0f9fb6c03f4b650d9/recovered.edits/0000000000000000020 at 1732519730795 (+2 ms)Processed 17 edits across 1 Regions in 19 ms; skipped=2; WAL=hdfs://localhost:44261/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732519730618/wal.1732519730626, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1732519730795 2024-11-25T07:28:50,814 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=409 (was 404) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:55178 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:46596 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=8358 (was 8354) - AvailableMemoryMB LEAK? - 2024-11-25T07:28:50,833 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=409, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=131, ProcessCount=11, AvailableMemoryMB=8349 2024-11-25T07:28:50,857 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:50,860 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:50,893 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T07:28:50,897 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-15140962, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/hregion-15140962, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:50,919 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-15140962/hregion-15140962.1732519730897, exclude list is [], retry=0 2024-11-25T07:28:50,924 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:50,924 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:50,924 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:50,932 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-15140962/hregion-15140962.1732519730897 2024-11-25T07:28:50,932 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:43429:43429)] 2024-11-25T07:28:50,932 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d10f1455b2e14585b5be1cfa9eb122cc, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44261/hbase 2024-11-25T07:28:50,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741879_1056 (size=70) 2024-11-25T07:28:50,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741879_1056 (size=70) 2024-11-25T07:28:50,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741879_1056 (size=70) 2024-11-25T07:28:50,947 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:50,952 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:50,954 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d10f1455b2e14585b5be1cfa9eb122cc columnFamilyName a 2024-11-25T07:28:50,954 DEBUG [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:50,955 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(327): Store=d10f1455b2e14585b5be1cfa9eb122cc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:50,955 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:50,957 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d10f1455b2e14585b5be1cfa9eb122cc columnFamilyName b 2024-11-25T07:28:50,957 DEBUG [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:50,957 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(327): Store=d10f1455b2e14585b5be1cfa9eb122cc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:50,957 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:50,959 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d10f1455b2e14585b5be1cfa9eb122cc columnFamilyName c 2024-11-25T07:28:50,959 DEBUG [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:50,959 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(327): Store=d10f1455b2e14585b5be1cfa9eb122cc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:50,959 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:50,960 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:50,964 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:50,966 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:50,966 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:50,966 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:28:50,968 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:50,970 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:28:50,971 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d10f1455b2e14585b5be1cfa9eb122cc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63973335, jitterRate=-0.04672302305698395}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:28:50,972 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d10f1455b2e14585b5be1cfa9eb122cc: Writing region info on filesystem at 1732519730947Initializing all the Stores at 1732519730948 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519730948Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519730952 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519730952Cleaning up temporary data from old regions at 1732519730966 (+14 ms)Region opened successfully at 1732519730972 (+6 ms) 2024-11-25T07:28:50,972 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d10f1455b2e14585b5be1cfa9eb122cc, disabling compactions & flushes 2024-11-25T07:28:50,972 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc. 2024-11-25T07:28:50,972 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc. 2024-11-25T07:28:50,972 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc. after waiting 0 ms 2024-11-25T07:28:50,972 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc. 2024-11-25T07:28:50,973 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc. 2024-11-25T07:28:50,973 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d10f1455b2e14585b5be1cfa9eb122cc: Waiting for close lock at 1732519730972Disabling compacts and flushes for region at 1732519730972Disabling writes for close at 1732519730972Writing region close event to WAL at 1732519730973 (+1 ms)Closed at 1732519730973 2024-11-25T07:28:50,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741878_1055 (size=93) 2024-11-25T07:28:50,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741878_1055 (size=93) 2024-11-25T07:28:50,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741878_1055 (size=93) 2024-11-25T07:28:50,982 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T07:28:50,982 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-15140962:(num 1732519730897) 2024-11-25T07:28:50,982 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:28:50,984 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:50,998 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984, exclude list is [], retry=0 2024-11-25T07:28:51,001 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:51,001 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:51,001 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:51,004 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984 2024-11-25T07:28:51,006 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:37263:37263)] 2024-11-25T07:28:51,007 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d10f1455b2e14585b5be1cfa9eb122cc, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:28:51,007 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:51,007 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:51,007 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:51,009 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:51,010 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d10f1455b2e14585b5be1cfa9eb122cc columnFamilyName a 2024-11-25T07:28:51,010 DEBUG [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:51,010 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(327): Store=d10f1455b2e14585b5be1cfa9eb122cc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:51,011 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:51,011 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d10f1455b2e14585b5be1cfa9eb122cc columnFamilyName b 2024-11-25T07:28:51,012 DEBUG [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:51,012 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(327): Store=d10f1455b2e14585b5be1cfa9eb122cc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:51,012 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:51,013 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d10f1455b2e14585b5be1cfa9eb122cc columnFamilyName c 2024-11-25T07:28:51,013 DEBUG [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:51,013 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(327): Store=d10f1455b2e14585b5be1cfa9eb122cc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:51,014 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:51,014 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:51,015 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:51,016 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:51,016 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:51,017 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:28:51,018 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:51,019 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d10f1455b2e14585b5be1cfa9eb122cc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66737469, jitterRate=-0.00553421676158905}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:28:51,020 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d10f1455b2e14585b5be1cfa9eb122cc: Writing region info on filesystem at 1732519731007Initializing all the Stores at 1732519731008 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519731008Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519731008Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519731009 (+1 ms)Cleaning up temporary data from old regions at 1732519731016 (+7 ms)Region opened successfully at 1732519731020 (+4 ms) 2024-11-25T07:28:51,025 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732519731024/Put/seqid=0 2024-11-25T07:28:51,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741881_1058 (size=4826) 2024-11-25T07:28:51,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741881_1058 (size=4826) 2024-11-25T07:28:51,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741881_1058 (size=4826) 2024-11-25T07:28:51,035 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44261/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in d10f1455b2e14585b5be1cfa9eb122cc/a 2024-11-25T07:28:51,044 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-25T07:28:51,044 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-25T07:28:51,044 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d10f1455b2e14585b5be1cfa9eb122cc: 2024-11-25T07:28:51,046 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/a/555604e98faa4db3ad9a0cfe14dc68cb_SeqId_3_ 2024-11-25T07:28:51,047 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44261/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into d10f1455b2e14585b5be1cfa9eb122cc/a as hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/a/555604e98faa4db3ad9a0cfe14dc68cb_SeqId_3_ - updating store file list. 2024-11-25T07:28:51,053 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 555604e98faa4db3ad9a0cfe14dc68cb_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-25T07:28:51,054 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/a/555604e98faa4db3ad9a0cfe14dc68cb_SeqId_3_ into d10f1455b2e14585b5be1cfa9eb122cc/a 2024-11-25T07:28:51,054 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44261/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into d10f1455b2e14585b5be1cfa9eb122cc/a (new location: hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/a/555604e98faa4db3ad9a0cfe14dc68cb_SeqId_3_) 2024-11-25T07:28:51,095 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44261/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984, size=0 (0bytes) 2024-11-25T07:28:51,095 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44261/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984 might be still open, length is 0 2024-11-25T07:28:51,095 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44261/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984 2024-11-25T07:28:51,095 WARN [IPC Server handler 4 on default port 44261 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-25T07:28:51,096 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44261/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984 after 1ms 2024-11-25T07:28:54,088 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:55734 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37043:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55734 dst: /127.0.0.1:37043 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37043 remote=/127.0.0.1:55734]. Total timeout mills is 60000, 56971 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:28:54,089 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:53476 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33603:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53476 dst: /127.0.0.1:33603 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:28:54,089 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:47528 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37227:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47528 dst: /127.0.0.1:37227 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:28:54,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741880_1059 (size=504) 2024-11-25T07:28:54,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741880_1059 (size=504) 2024-11-25T07:28:55,096 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44261/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984 after 4001ms 2024-11-25T07:28:55,100 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44261/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984 took 4006ms 2024-11-25T07:28:55,102 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44261/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984; continuing. 2024-11-25T07:28:55,102 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44261/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984 so closing down 2024-11-25T07:28:55,102 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T07:28:55,104 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732519730984.temp 2024-11-25T07:28:55,105 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/recovered.edits/0000000000000000005-wal.1732519730984.temp 2024-11-25T07:28:55,105 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T07:28:55,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741882_1060 (size=238) 2024-11-25T07:28:55,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741882_1060 (size=238) 2024-11-25T07:28:55,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741882_1060 (size=238) 2024-11-25T07:28:55,113 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/recovered.edits/0000000000000000005-wal.1732519730984.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-25T07:28:55,114 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/recovered.edits/0000000000000000005-wal.1732519730984.temp to hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/recovered.edits/0000000000000000005 2024-11-25T07:28:55,115 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 15 ms; skipped=1; WAL=hdfs://localhost:44261/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984, size=0, length=0, corrupted=false, cancelled=false 2024-11-25T07:28:55,115 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44261/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984, journal: Splitting hdfs://localhost:44261/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984, size=0 (0bytes) at 1732519731095Finishing writing output for hdfs://localhost:44261/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984 so closing down at 1732519735102 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/recovered.edits/0000000000000000005-wal.1732519730984.temp at 1732519735105 (+3 ms)3 split writer threads finished at 1732519735105Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/recovered.edits/0000000000000000005-wal.1732519730984.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732519735113 (+8 ms)Rename recovered edits hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/recovered.edits/0000000000000000005-wal.1732519730984.temp to hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/recovered.edits/0000000000000000005 at 1732519735115 (+2 ms)Processed 2 edits across 1 Regions in 15 ms; skipped=1; WAL=hdfs://localhost:44261/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984, size=0, length=0, corrupted=false, cancelled=false at 1732519735115 2024-11-25T07:28:55,116 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44261/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984 to hdfs://localhost:44261/hbase/oldWALs/wal.1732519730984 2024-11-25T07:28:55,117 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/recovered.edits/0000000000000000005 2024-11-25T07:28:55,118 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:28:55,119 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:55,133 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519735120, exclude list is [], retry=0 2024-11-25T07:28:55,136 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:55,136 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:55,136 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:55,138 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519735120 2024-11-25T07:28:55,138 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:43429:43429)] 2024-11-25T07:28:55,138 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d10f1455b2e14585b5be1cfa9eb122cc, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:28:55,138 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:55,139 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:55,139 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:55,140 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:55,141 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d10f1455b2e14585b5be1cfa9eb122cc columnFamilyName a 2024-11-25T07:28:55,141 DEBUG [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:55,148 DEBUG [StoreFileOpener-d10f1455b2e14585b5be1cfa9eb122cc-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 555604e98faa4db3ad9a0cfe14dc68cb_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-25T07:28:55,148 DEBUG [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/a/555604e98faa4db3ad9a0cfe14dc68cb_SeqId_3_ 2024-11-25T07:28:55,149 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(327): Store=d10f1455b2e14585b5be1cfa9eb122cc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:55,149 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:55,150 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d10f1455b2e14585b5be1cfa9eb122cc columnFamilyName b 2024-11-25T07:28:55,150 DEBUG [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:55,150 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(327): Store=d10f1455b2e14585b5be1cfa9eb122cc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:55,150 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:55,151 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d10f1455b2e14585b5be1cfa9eb122cc columnFamilyName c 2024-11-25T07:28:55,151 DEBUG [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:55,151 INFO [StoreOpener-d10f1455b2e14585b5be1cfa9eb122cc-1 {}] regionserver.HStore(327): Store=d10f1455b2e14585b5be1cfa9eb122cc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:55,151 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:55,152 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:55,154 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:55,154 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/recovered.edits/0000000000000000005 2024-11-25T07:28:55,157 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/recovered.edits/0000000000000000005 2024-11-25T07:28:55,157 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d10f1455b2e14585b5be1cfa9eb122cc 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-25T07:28:55,172 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/.tmp/a/7824110b44b24b5d81b29b3fa03614d3 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732519731059/Put/seqid=0 2024-11-25T07:28:55,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741884_1062 (size=5149) 2024-11-25T07:28:55,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741884_1062 (size=5149) 2024-11-25T07:28:55,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741884_1062 (size=5149) 2024-11-25T07:28:55,179 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/.tmp/a/7824110b44b24b5d81b29b3fa03614d3 2024-11-25T07:28:55,185 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/.tmp/a/7824110b44b24b5d81b29b3fa03614d3 as hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/a/7824110b44b24b5d81b29b3fa03614d3 2024-11-25T07:28:55,190 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/a/7824110b44b24b5d81b29b3fa03614d3, entries=1, sequenceid=5, filesize=5.0 K 2024-11-25T07:28:55,191 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for d10f1455b2e14585b5be1cfa9eb122cc in 34ms, sequenceid=5, compaction requested=false; wal=null 2024-11-25T07:28:55,191 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/recovered.edits/0000000000000000005 2024-11-25T07:28:55,192 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:55,192 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:55,193 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:28:55,195 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d10f1455b2e14585b5be1cfa9eb122cc 2024-11-25T07:28:55,197 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d10f1455b2e14585b5be1cfa9eb122cc/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-25T07:28:55,198 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d10f1455b2e14585b5be1cfa9eb122cc; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66331834, jitterRate=-0.011578649282455444}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:28:55,198 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d10f1455b2e14585b5be1cfa9eb122cc: Writing region info on filesystem at 1732519735139Initializing all the Stores at 1732519735139Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519735140 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519735140Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519735140Obtaining lock to block concurrent updates at 1732519735157 (+17 ms)Preparing flush snapshotting stores in d10f1455b2e14585b5be1cfa9eb122cc at 1732519735157Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732519735157Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc. at 1732519735157Flushing d10f1455b2e14585b5be1cfa9eb122cc/a: creating writer at 1732519735157Flushing d10f1455b2e14585b5be1cfa9eb122cc/a: appending metadata at 1732519735171 (+14 ms)Flushing d10f1455b2e14585b5be1cfa9eb122cc/a: closing flushed file at 1732519735171Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57965038: reopening flushed file at 1732519735184 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for d10f1455b2e14585b5be1cfa9eb122cc in 34ms, sequenceid=5, compaction requested=false; wal=null at 1732519735191 (+7 ms)Cleaning up temporary data from old regions at 1732519735193 (+2 ms)Region opened successfully at 1732519735198 (+5 ms) 2024-11-25T07:28:55,202 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d10f1455b2e14585b5be1cfa9eb122cc, disabling compactions & flushes 2024-11-25T07:28:55,202 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc. 2024-11-25T07:28:55,202 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc. 2024-11-25T07:28:55,202 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc. after waiting 0 ms 2024-11-25T07:28:55,202 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc. 2024-11-25T07:28:55,203 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732519730858.d10f1455b2e14585b5be1cfa9eb122cc. 2024-11-25T07:28:55,203 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d10f1455b2e14585b5be1cfa9eb122cc: Waiting for close lock at 1732519735202Disabling compacts and flushes for region at 1732519735202Disabling writes for close at 1732519735202Writing region close event to WAL at 1732519735203 (+1 ms)Closed at 1732519735203 2024-11-25T07:28:55,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741883_1061 (size=93) 2024-11-25T07:28:55,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741883_1061 (size=93) 2024-11-25T07:28:55,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741883_1061 (size=93) 2024-11-25T07:28:55,208 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T07:28:55,208 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732519735120) 2024-11-25T07:28:55,221 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=414 (was 409) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2083721719_22 at /127.0.0.1:47552 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:44261 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2083721719_22 at /127.0.0.1:53494 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2083721719_22 at /127.0.0.1:55786 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (473829079) connection to localhost/127.0.0.1:44261 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=121 (was 131), ProcessCount=11 (was 11), AvailableMemoryMB=8347 (was 8349) 2024-11-25T07:28:55,233 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=414, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=121, ProcessCount=11, AvailableMemoryMB=8347 2024-11-25T07:28:55,248 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:55,253 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T07:28:55,258 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5eb3d201e8c9,33903,1732519720118 2024-11-25T07:28:55,260 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@641f2200 2024-11-25T07:28:55,261 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T07:28:55,263 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T07:28:55,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33903 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T07:28:55,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33903 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-25T07:28:55,274 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T07:28:55,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33903 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-25T07:28:55,277 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:55,278 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T07:28:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T07:28:55,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741885_1063 (size=694) 2024-11-25T07:28:55,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741885_1063 (size=694) 2024-11-25T07:28:55,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741885_1063 (size=694) 2024-11-25T07:28:55,291 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0197c07d98552bf4cd477bc9d94b6b62, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438 2024-11-25T07:28:55,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741886_1064 (size=77) 2024-11-25T07:28:55,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741886_1064 (size=77) 2024-11-25T07:28:55,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741886_1064 (size=77) 2024-11-25T07:28:55,299 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:55,299 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 0197c07d98552bf4cd477bc9d94b6b62, disabling compactions & flushes 2024-11-25T07:28:55,299 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:55,300 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:55,300 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. after waiting 0 ms 2024-11-25T07:28:55,300 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:55,300 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:55,300 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0197c07d98552bf4cd477bc9d94b6b62: Waiting for close lock at 1732519735299Disabling compacts and flushes for region at 1732519735299Disabling writes for close at 1732519735300 (+1 ms)Writing region close event to WAL at 1732519735300Closed at 1732519735300 2024-11-25T07:28:55,301 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T07:28:55,306 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732519735301"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732519735301"}]},"ts":"1732519735301"} 2024-11-25T07:28:55,309 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T07:28:55,310 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T07:28:55,313 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732519735310"}]},"ts":"1732519735310"} 2024-11-25T07:28:55,317 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-25T07:28:55,317 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {5eb3d201e8c9=0} racks are {/default-rack=0} 2024-11-25T07:28:55,318 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T07:28:55,318 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T07:28:55,318 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-25T07:28:55,318 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T07:28:55,319 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T07:28:55,319 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-25T07:28:55,319 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T07:28:55,319 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T07:28:55,319 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-25T07:28:55,319 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T07:28:55,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, ASSIGN}] 2024-11-25T07:28:55,322 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, ASSIGN 2024-11-25T07:28:55,323 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, ASSIGN; state=OFFLINE, location=5eb3d201e8c9,35771,1732519720972; forceNewPlan=false, retain=false 2024-11-25T07:28:55,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T07:28:55,476 INFO [5eb3d201e8c9:33903 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T07:28:55,477 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0197c07d98552bf4cd477bc9d94b6b62, regionState=OPENING, regionLocation=5eb3d201e8c9,35771,1732519720972 2024-11-25T07:28:55,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, ASSIGN because future has completed 2024-11-25T07:28:55,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,35771,1732519720972}] 2024-11-25T07:28:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T07:28:55,634 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T07:28:55,636 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54033, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T07:28:55,641 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:55,642 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0197c07d98552bf4cd477bc9d94b6b62, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:28:55,642 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:55,642 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:55,642 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:55,642 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:55,644 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:55,645 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0197c07d98552bf4cd477bc9d94b6b62 columnFamilyName cf1 2024-11-25T07:28:55,645 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:55,646 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(327): Store=0197c07d98552bf4cd477bc9d94b6b62/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:55,646 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:55,647 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0197c07d98552bf4cd477bc9d94b6b62 columnFamilyName cf2 2024-11-25T07:28:55,647 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:55,648 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(327): Store=0197c07d98552bf4cd477bc9d94b6b62/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:55,648 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:55,649 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:55,649 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:55,650 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:55,650 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:55,651 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T07:28:55,652 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:55,654 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:28:55,655 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0197c07d98552bf4cd477bc9d94b6b62; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62941221, jitterRate=-0.06210272014141083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T07:28:55,655 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:55,656 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0197c07d98552bf4cd477bc9d94b6b62: Running coprocessor pre-open hook at 1732519735642Writing region info on filesystem at 1732519735642Initializing all the Stores at 1732519735643 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519735643Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519735644 (+1 ms)Cleaning up temporary data from old regions at 1732519735650 (+6 ms)Running coprocessor post-open hooks at 1732519735655 (+5 ms)Region opened successfully at 1732519735656 (+1 ms) 2024-11-25T07:28:55,657 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., pid=6, masterSystemTime=1732519735634 2024-11-25T07:28:55,660 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:55,660 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:55,662 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0197c07d98552bf4cd477bc9d94b6b62, regionState=OPEN, openSeqNum=2, regionLocation=5eb3d201e8c9,35771,1732519720972 2024-11-25T07:28:55,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,35771,1732519720972 because future has completed 2024-11-25T07:28:55,669 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T07:28:55,669 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,35771,1732519720972 in 185 msec 2024-11-25T07:28:55,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T07:28:55,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, ASSIGN in 350 msec 2024-11-25T07:28:55,675 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T07:28:55,675 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732519735675"}]},"ts":"1732519735675"} 2024-11-25T07:28:55,678 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-25T07:28:55,679 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T07:28:55,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 411 msec 2024-11-25T07:28:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T07:28:55,915 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-25T07:28:55,915 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-25T07:28:55,916 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:28:55,924 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-25T07:28:55,924 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:28:55,925 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-25T07:28:55,939 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=2] 2024-11-25T07:28:55,941 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T07:28:55,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T07:28:55,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33903 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=0197c07d98552bf4cd477bc9d94b6b62, source=5eb3d201e8c9,35771,1732519720972, destination=5eb3d201e8c9,33753,1732519720813, warming up region on 5eb3d201e8c9,33753,1732519720813 2024-11-25T07:28:55,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33903 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=0197c07d98552bf4cd477bc9d94b6b62, source=5eb3d201e8c9,35771,1732519720972, destination=5eb3d201e8c9,33753,1732519720813, running balancer 2024-11-25T07:28:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33903 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, REOPEN/MOVE 2024-11-25T07:28:55,961 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, REOPEN/MOVE 2024-11-25T07:28:55,963 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0197c07d98552bf4cd477bc9d94b6b62, regionState=CLOSING, regionLocation=5eb3d201e8c9,35771,1732519720972 2024-11-25T07:28:55,964 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33753 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33753 {}] regionserver.HRegion(7855): Warmup {ENCODED => 0197c07d98552bf4cd477bc9d94b6b62, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:28:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33753 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:55,965 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:55,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, REOPEN/MOVE because future has completed 2024-11-25T07:28:55,966 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0197c07d98552bf4cd477bc9d94b6b62 columnFamilyName cf1 2024-11-25T07:28:55,967 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:55,967 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T07:28:55,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,35771,1732519720972}] 2024-11-25T07:28:55,967 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(327): Store=0197c07d98552bf4cd477bc9d94b6b62/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:55,967 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:55,968 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0197c07d98552bf4cd477bc9d94b6b62 columnFamilyName cf2 2024-11-25T07:28:55,968 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:55,969 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(327): Store=0197c07d98552bf4cd477bc9d94b6b62/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33753 {}] regionserver.HRegion(1722): Closing 0197c07d98552bf4cd477bc9d94b6b62, disabling compactions & flushes 2024-11-25T07:28:55,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33753 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33753 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33753 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. after waiting 0 ms 2024-11-25T07:28:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33753 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:55,970 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33753 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33753 {}] regionserver.HRegion(1676): Region close journal for 0197c07d98552bf4cd477bc9d94b6b62: Waiting for close lock at 1732519735969Disabling compacts and flushes for region at 1732519735969Disabling writes for close at 1732519735969Writing region close event to WAL at 1732519735970 (+1 ms)Closed at 1732519735970 2024-11-25T07:28:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33903 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-25T07:28:56,125 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:56,126 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T07:28:56,126 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 0197c07d98552bf4cd477bc9d94b6b62, disabling compactions & flushes 2024-11-25T07:28:56,126 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:56,126 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:56,126 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. after waiting 0 ms 2024-11-25T07:28:56,126 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:56,127 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 0197c07d98552bf4cd477bc9d94b6b62 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-25T07:28:56,147 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/.tmp/cf1/c6b17cd734064af1986f82b10709312a is 35, key is r1/cf1:q/1732519735943/Put/seqid=0 2024-11-25T07:28:56,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741887_1065 (size=4783) 2024-11-25T07:28:56,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741887_1065 (size=4783) 2024-11-25T07:28:56,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741887_1065 (size=4783) 2024-11-25T07:28:56,155 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/.tmp/cf1/c6b17cd734064af1986f82b10709312a 2024-11-25T07:28:56,162 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/.tmp/cf1/c6b17cd734064af1986f82b10709312a as hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/c6b17cd734064af1986f82b10709312a 2024-11-25T07:28:56,169 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/c6b17cd734064af1986f82b10709312a, entries=1, sequenceid=5, filesize=4.7 K 2024-11-25T07:28:56,170 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 0197c07d98552bf4cd477bc9d94b6b62 in 44ms, sequenceid=5, compaction requested=false 2024-11-25T07:28:56,170 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-25T07:28:56,175 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-25T07:28:56,178 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:56,178 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 0197c07d98552bf4cd477bc9d94b6b62: Waiting for close lock at 1732519736126Running coprocessor pre-close hooks at 1732519736126Disabling compacts and flushes for region at 1732519736126Disabling writes for close at 1732519736126Obtaining lock to block concurrent updates at 1732519736127 (+1 ms)Preparing flush snapshotting stores in 0197c07d98552bf4cd477bc9d94b6b62 at 1732519736127Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732519736127Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. at 1732519736128 (+1 ms)Flushing 0197c07d98552bf4cd477bc9d94b6b62/cf1: creating writer at 1732519736128Flushing 0197c07d98552bf4cd477bc9d94b6b62/cf1: appending metadata at 1732519736147 (+19 ms)Flushing 0197c07d98552bf4cd477bc9d94b6b62/cf1: closing flushed file at 1732519736147Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e01b5d0: reopening flushed file at 1732519736161 (+14 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 0197c07d98552bf4cd477bc9d94b6b62 in 44ms, sequenceid=5, compaction requested=false at 1732519736170 (+9 ms)Writing region close event to WAL at 1732519736172 (+2 ms)Running coprocessor post-close hooks at 1732519736176 (+4 ms)Closed at 1732519736178 (+2 ms) 2024-11-25T07:28:56,179 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 0197c07d98552bf4cd477bc9d94b6b62 move to 5eb3d201e8c9,33753,1732519720813 record at close sequenceid=5 2024-11-25T07:28:56,182 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:56,183 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0197c07d98552bf4cd477bc9d94b6b62, regionState=CLOSED 2024-11-25T07:28:56,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,35771,1732519720972 because future has completed 2024-11-25T07:28:56,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T07:28:56,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,35771,1732519720972 in 220 msec 2024-11-25T07:28:56,191 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, REOPEN/MOVE; state=CLOSED, location=5eb3d201e8c9,33753,1732519720813; forceNewPlan=false, retain=false 2024-11-25T07:28:56,341 INFO [5eb3d201e8c9:33903 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T07:28:56,342 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0197c07d98552bf4cd477bc9d94b6b62, regionState=OPENING, regionLocation=5eb3d201e8c9,33753,1732519720813 2024-11-25T07:28:56,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, REOPEN/MOVE because future has completed 2024-11-25T07:28:56,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,33753,1732519720813}] 2024-11-25T07:28:56,502 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:56,503 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 0197c07d98552bf4cd477bc9d94b6b62, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:28:56,503 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:56,503 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:56,503 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:56,503 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:56,505 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:56,506 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0197c07d98552bf4cd477bc9d94b6b62 columnFamilyName cf1 2024-11-25T07:28:56,506 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:56,512 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/c6b17cd734064af1986f82b10709312a 2024-11-25T07:28:56,512 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(327): Store=0197c07d98552bf4cd477bc9d94b6b62/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:56,512 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:56,513 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0197c07d98552bf4cd477bc9d94b6b62 columnFamilyName cf2 2024-11-25T07:28:56,513 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:56,514 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(327): Store=0197c07d98552bf4cd477bc9d94b6b62/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:56,514 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:56,515 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:56,516 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:56,517 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:56,517 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:56,517 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T07:28:56,519 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:56,520 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 0197c07d98552bf4cd477bc9d94b6b62; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73684811, jitterRate=0.09798924624919891}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T07:28:56,520 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:56,521 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 0197c07d98552bf4cd477bc9d94b6b62: Running coprocessor pre-open hook at 1732519736503Writing region info on filesystem at 1732519736503Initializing all the Stores at 1732519736504 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519736504Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519736505 (+1 ms)Cleaning up temporary data from old regions at 1732519736517 (+12 ms)Running coprocessor post-open hooks at 1732519736520 (+3 ms)Region opened successfully at 1732519736520 2024-11-25T07:28:56,522 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., pid=9, masterSystemTime=1732519736498 2024-11-25T07:28:56,524 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:56,524 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:56,525 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0197c07d98552bf4cd477bc9d94b6b62, regionState=OPEN, openSeqNum=9, regionLocation=5eb3d201e8c9,33753,1732519720813 2024-11-25T07:28:56,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,33753,1732519720813 because future has completed 2024-11-25T07:28:56,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-25T07:28:56,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,33753,1732519720813 in 184 msec 2024-11-25T07:28:56,534 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, REOPEN/MOVE in 573 msec 2024-11-25T07:28:56,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T07:28:56,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T07:28:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35771 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:49020 deadline: 1732519796573, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5eb3d201e8c9 port=33753 startCode=1732519720813. As of locationSeqNum=5. 2024-11-25T07:28:56,579 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5eb3d201e8c9 port=33753 startCode=1732519720813. As of locationSeqNum=5. 2024-11-25T07:28:56,579 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5eb3d201e8c9 port=33753 startCode=1732519720813. As of locationSeqNum=5. 2024-11-25T07:28:56,580 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,33753,1732519720813, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5eb3d201e8c9 port=33753 startCode=1732519720813. As of locationSeqNum=5. 2024-11-25T07:28:56,695 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0197c07d98552bf4cd477bc9d94b6b62 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-25T07:28:56,720 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/.tmp/cf1/462152e9aade42b19595dec491de01d5 is 29, key is r1/cf1:/1732519736686/DeleteFamily/seqid=0 2024-11-25T07:28:56,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741888_1066 (size=4906) 2024-11-25T07:28:56,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741888_1066 (size=4906) 2024-11-25T07:28:56,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741888_1066 (size=4906) 2024-11-25T07:28:56,730 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/.tmp/cf1/462152e9aade42b19595dec491de01d5 2024-11-25T07:28:56,737 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 462152e9aade42b19595dec491de01d5 2024-11-25T07:28:56,760 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/.tmp/cf2/47b380ff90824dc7b6a94f19be01061d is 29, key is r1/cf2:/1732519736686/DeleteFamily/seqid=0 2024-11-25T07:28:56,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741889_1067 (size=4906) 2024-11-25T07:28:56,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741889_1067 (size=4906) 2024-11-25T07:28:56,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741889_1067 (size=4906) 2024-11-25T07:28:56,769 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/.tmp/cf2/47b380ff90824dc7b6a94f19be01061d 2024-11-25T07:28:56,776 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 47b380ff90824dc7b6a94f19be01061d 2024-11-25T07:28:56,777 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/.tmp/cf1/462152e9aade42b19595dec491de01d5 as hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/462152e9aade42b19595dec491de01d5 2024-11-25T07:28:56,784 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 462152e9aade42b19595dec491de01d5 2024-11-25T07:28:56,784 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/462152e9aade42b19595dec491de01d5, entries=1, sequenceid=12, filesize=4.8 K 2024-11-25T07:28:56,785 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/.tmp/cf2/47b380ff90824dc7b6a94f19be01061d as hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf2/47b380ff90824dc7b6a94f19be01061d 2024-11-25T07:28:56,792 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 47b380ff90824dc7b6a94f19be01061d 2024-11-25T07:28:56,792 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf2/47b380ff90824dc7b6a94f19be01061d, entries=1, sequenceid=12, filesize=4.8 K 2024-11-25T07:28:56,793 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 0197c07d98552bf4cd477bc9d94b6b62 in 98ms, sequenceid=12, compaction requested=false 2024-11-25T07:28:56,793 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0197c07d98552bf4cd477bc9d94b6b62: 2024-11-25T07:28:56,797 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T07:28:56,799 DEBUG [Time-limited test {}] regionserver.HStore(1541): 0197c07d98552bf4cd477bc9d94b6b62/cf1 is initiating major compaction (all files) 2024-11-25T07:28:56,799 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T07:28:56,799 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:56,800 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 0197c07d98552bf4cd477bc9d94b6b62/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:56,800 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/c6b17cd734064af1986f82b10709312a, hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/462152e9aade42b19595dec491de01d5] into tmpdir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/.tmp, totalSize=9.5 K 2024-11-25T07:28:56,801 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c6b17cd734064af1986f82b10709312a, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732519735943 2024-11-25T07:28:56,802 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 462152e9aade42b19595dec491de01d5, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-25T07:28:56,816 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 0197c07d98552bf4cd477bc9d94b6b62#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T07:28:56,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741890_1068 (size=4626) 2024-11-25T07:28:56,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741890_1068 (size=4626) 2024-11-25T07:28:56,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741890_1068 (size=4626) 2024-11-25T07:28:56,833 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/.tmp/cf1/6316b588bbde43c8b43459e5d631fde2 as hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/6316b588bbde43c8b43459e5d631fde2 2024-11-25T07:28:56,847 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 0197c07d98552bf4cd477bc9d94b6b62/cf1 of 0197c07d98552bf4cd477bc9d94b6b62 into 6316b588bbde43c8b43459e5d631fde2(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T07:28:56,847 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 0197c07d98552bf4cd477bc9d94b6b62: 2024-11-25T07:28:56,847 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-25T07:28:56,847 DEBUG [Time-limited test {}] regionserver.HStore(1541): 0197c07d98552bf4cd477bc9d94b6b62/cf2 is initiating major compaction (all files) 2024-11-25T07:28:56,847 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T07:28:56,848 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T07:28:56,848 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 0197c07d98552bf4cd477bc9d94b6b62/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:56,848 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf2/47b380ff90824dc7b6a94f19be01061d] into tmpdir=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/.tmp, totalSize=4.8 K 2024-11-25T07:28:56,848 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 47b380ff90824dc7b6a94f19be01061d, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-25T07:28:56,854 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 0197c07d98552bf4cd477bc9d94b6b62#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T07:28:56,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741891_1069 (size=4592) 2024-11-25T07:28:56,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741891_1069 (size=4592) 2024-11-25T07:28:56,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741891_1069 (size=4592) 2024-11-25T07:28:56,870 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/.tmp/cf2/5b9a05600f0540289e73c40ade72246b as hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf2/5b9a05600f0540289e73c40ade72246b 2024-11-25T07:28:56,877 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 0197c07d98552bf4cd477bc9d94b6b62/cf2 of 0197c07d98552bf4cd477bc9d94b6b62 into 5b9a05600f0540289e73c40ade72246b(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T07:28:56,877 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 0197c07d98552bf4cd477bc9d94b6b62: 2024-11-25T07:28:56,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33903 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=0197c07d98552bf4cd477bc9d94b6b62, source=5eb3d201e8c9,33753,1732519720813, destination=5eb3d201e8c9,35771,1732519720972, warming up region on 5eb3d201e8c9,35771,1732519720972 2024-11-25T07:28:56,883 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33903 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=0197c07d98552bf4cd477bc9d94b6b62, source=5eb3d201e8c9,33753,1732519720813, destination=5eb3d201e8c9,35771,1732519720972, running balancer 2024-11-25T07:28:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33903 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, REOPEN/MOVE 2024-11-25T07:28:56,884 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, REOPEN/MOVE 2024-11-25T07:28:56,886 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=0197c07d98552bf4cd477bc9d94b6b62, regionState=CLOSING, regionLocation=5eb3d201e8c9,33753,1732519720813 2024-11-25T07:28:56,886 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35771 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35771 {}] regionserver.HRegion(7855): Warmup {ENCODED => 0197c07d98552bf4cd477bc9d94b6b62, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:28:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35771 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:56,887 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:56,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, REOPEN/MOVE because future has completed 2024-11-25T07:28:56,889 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T07:28:56,889 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0197c07d98552bf4cd477bc9d94b6b62 columnFamilyName cf1 2024-11-25T07:28:56,889 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:56,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,33753,1732519720813}] 2024-11-25T07:28:56,897 INFO [StoreFileOpener-0197c07d98552bf4cd477bc9d94b6b62-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 462152e9aade42b19595dec491de01d5 2024-11-25T07:28:56,897 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/462152e9aade42b19595dec491de01d5 2024-11-25T07:28:56,902 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/6316b588bbde43c8b43459e5d631fde2 2024-11-25T07:28:56,907 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/c6b17cd734064af1986f82b10709312a 2024-11-25T07:28:56,907 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(327): Store=0197c07d98552bf4cd477bc9d94b6b62/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:56,907 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:56,909 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0197c07d98552bf4cd477bc9d94b6b62 columnFamilyName cf2 2024-11-25T07:28:56,909 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:56,915 INFO [StoreFileOpener-0197c07d98552bf4cd477bc9d94b6b62-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 47b380ff90824dc7b6a94f19be01061d 2024-11-25T07:28:56,915 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf2/47b380ff90824dc7b6a94f19be01061d 2024-11-25T07:28:56,920 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf2/5b9a05600f0540289e73c40ade72246b 2024-11-25T07:28:56,921 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(327): Store=0197c07d98552bf4cd477bc9d94b6b62/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35771 {}] regionserver.HRegion(1722): Closing 0197c07d98552bf4cd477bc9d94b6b62, disabling compactions & flushes 2024-11-25T07:28:56,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35771 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35771 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35771 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. after waiting 0 ms 2024-11-25T07:28:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35771 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:56,922 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35771 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35771 {}] regionserver.HRegion(1676): Region close journal for 0197c07d98552bf4cd477bc9d94b6b62: Waiting for close lock at 1732519736921Disabling compacts and flushes for region at 1732519736921Disabling writes for close at 1732519736921Writing region close event to WAL at 1732519736922 (+1 ms)Closed at 1732519736922 2024-11-25T07:28:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33903 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-25T07:28:57,043 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:57,043 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T07:28:57,043 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 0197c07d98552bf4cd477bc9d94b6b62, disabling compactions & flushes 2024-11-25T07:28:57,043 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:57,043 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:57,043 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. after waiting 0 ms 2024-11-25T07:28:57,043 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:57,044 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/c6b17cd734064af1986f82b10709312a, hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/462152e9aade42b19595dec491de01d5] to archive 2024-11-25T07:28:57,047 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T07:28:57,050 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/c6b17cd734064af1986f82b10709312a to hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/c6b17cd734064af1986f82b10709312a 2024-11-25T07:28:57,052 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/462152e9aade42b19595dec491de01d5 to hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/462152e9aade42b19595dec491de01d5 2024-11-25T07:28:57,065 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf2/47b380ff90824dc7b6a94f19be01061d] to archive 2024-11-25T07:28:57,066 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T07:28:57,068 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf2/47b380ff90824dc7b6a94f19be01061d to hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf2/47b380ff90824dc7b6a94f19be01061d 2024-11-25T07:28:57,073 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-25T07:28:57,074 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:57,074 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 0197c07d98552bf4cd477bc9d94b6b62: Waiting for close lock at 1732519737043Running coprocessor pre-close hooks at 1732519737043Disabling compacts and flushes for region at 1732519737043Disabling writes for close at 1732519737043Writing region close event to WAL at 1732519737069 (+26 ms)Running coprocessor post-close hooks at 1732519737073 (+4 ms)Closed at 1732519737073 2024-11-25T07:28:57,074 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 0197c07d98552bf4cd477bc9d94b6b62 move to 5eb3d201e8c9,35771,1732519720972 record at close sequenceid=12 2024-11-25T07:28:57,076 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:57,077 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=0197c07d98552bf4cd477bc9d94b6b62, regionState=CLOSED 2024-11-25T07:28:57,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,33753,1732519720813 because future has completed 2024-11-25T07:28:57,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-25T07:28:57,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,33753,1732519720813 in 191 msec 2024-11-25T07:28:57,084 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, REOPEN/MOVE; state=CLOSED, location=5eb3d201e8c9,35771,1732519720972; forceNewPlan=false, retain=false 2024-11-25T07:28:57,234 INFO [5eb3d201e8c9:33903 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T07:28:57,235 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=0197c07d98552bf4cd477bc9d94b6b62, regionState=OPENING, regionLocation=5eb3d201e8c9,35771,1732519720972 2024-11-25T07:28:57,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, REOPEN/MOVE because future has completed 2024-11-25T07:28:57,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,35771,1732519720972}] 2024-11-25T07:28:57,395 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:57,395 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 0197c07d98552bf4cd477bc9d94b6b62, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:28:57,395 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:57,395 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:57,395 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:57,395 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:57,397 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:57,398 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0197c07d98552bf4cd477bc9d94b6b62 columnFamilyName cf1 2024-11-25T07:28:57,398 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:57,404 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/6316b588bbde43c8b43459e5d631fde2 2024-11-25T07:28:57,404 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(327): Store=0197c07d98552bf4cd477bc9d94b6b62/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:57,405 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:57,405 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0197c07d98552bf4cd477bc9d94b6b62 columnFamilyName cf2 2024-11-25T07:28:57,405 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:57,411 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf2/5b9a05600f0540289e73c40ade72246b 2024-11-25T07:28:57,411 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(327): Store=0197c07d98552bf4cd477bc9d94b6b62/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:57,411 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:57,412 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:57,413 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:57,414 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:57,414 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:57,415 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T07:28:57,417 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:57,418 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 0197c07d98552bf4cd477bc9d94b6b62; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61824365, jitterRate=-0.07874517142772675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T07:28:57,418 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:57,419 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 0197c07d98552bf4cd477bc9d94b6b62: Running coprocessor pre-open hook at 1732519737396Writing region info on filesystem at 1732519737396Initializing all the Stores at 1732519737396Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519737396Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519737397 (+1 ms)Cleaning up temporary data from old regions at 1732519737414 (+17 ms)Running coprocessor post-open hooks at 1732519737418 (+4 ms)Region opened successfully at 1732519737419 (+1 ms) 2024-11-25T07:28:57,420 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., pid=12, masterSystemTime=1732519737390 2024-11-25T07:28:57,423 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:57,423 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:57,424 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=0197c07d98552bf4cd477bc9d94b6b62, regionState=OPEN, openSeqNum=18, regionLocation=5eb3d201e8c9,35771,1732519720972 2024-11-25T07:28:57,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,35771,1732519720972 because future has completed 2024-11-25T07:28:57,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-25T07:28:57,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,35771,1732519720972 in 189 msec 2024-11-25T07:28:57,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, REOPEN/MOVE in 547 msec 2024-11-25T07:28:57,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T07:28:57,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T07:28:57,439 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 5eb3d201e8c9,35771,1732519720972: testing ***** 2024-11-25T07:28:57,439 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-25T07:28:57,442 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-25T07:28:57,444 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-25T07:28:57,448 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-25T07:28:57,449 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-25T07:28:57,458 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 392069096 }, "NonHeapMemoryUsage": { "committed": 170328064, "init": 7667712, "max": -1, "used": 167763880 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "5eb3d201e8c9", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 16, "ProcessCallTime_mean": 5, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 8, "ProcessCallTime_75th_percentile": 12, "ProcessCallTime_90th_percentile": 14, "ProcessCallTime_95th_percentile": 15, "ProcessCallTime_98th_percentile": 15, "ProcessCallTime_99th_percentile": 15, "ProcessCallTime_99.9th_percentile": 15, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 0, "TotalCallTime_max": 16, "TotalCallTime_mean": 5, "TotalCallTime_25th_percentile": 4, "TotalCallTime_median": 8, "TotalCallTime_75th_percentile": 12, "TotalCallTime_90th_percentile": 14, "TotalCallTime_95th_percentile": 15, "TotalCallTime_98th_percentile": 15, "TotalCallTime_99th_percentile": 15, "TotalCallTime_99.9th_percentile": 15, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "5eb3d201e8c9", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:50797", "tag.serverName": "5eb3d201e8c9,44311,1732519720918", "tag.clusterId": "f004f047-221d-470c-b151-81737d8f21a0", "tag.Context": "regionserver", "tag.Hostname": "5eb3d201e8c9", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732519720918, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-25T07:28:57,462 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33903 {}] master.MasterRpcServices(700): 5eb3d201e8c9,35771,1732519720972 reported a fatal error: ***** ABORTING region server 5eb3d201e8c9,35771,1732519720972: testing ***** 2024-11-25T07:28:57,465 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5eb3d201e8c9,35771,1732519720972' ***** 2024-11-25T07:28:57,465 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-25T07:28:57,465 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T07:28:57,465 INFO [RS:2;5eb3d201e8c9:35771 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-25T07:28:57,465 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T07:28:57,466 INFO [RS:2;5eb3d201e8c9:35771 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-25T07:28:57,466 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(3091): Received CLOSE for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:57,466 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(956): aborting server 5eb3d201e8c9,35771,1732519720972 2024-11-25T07:28:57,466 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T07:28:57,466 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0197c07d98552bf4cd477bc9d94b6b62, disabling compactions & flushes 2024-11-25T07:28:57,466 INFO [RS:2;5eb3d201e8c9:35771 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;5eb3d201e8c9:35771. 2024-11-25T07:28:57,466 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:59646 deadline: 1732519797466, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5eb3d201e8c9 port=35771 startCode=1732519720972. As of locationSeqNum=12. 2024-11-25T07:28:57,467 DEBUG [RS:2;5eb3d201e8c9:35771 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T07:28:57,467 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:57,467 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. after waiting 0 ms 2024-11-25T07:28:57,467 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:57,467 DEBUG [RS:2;5eb3d201e8c9:35771 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:28:57,467 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T07:28:57,467 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,33753,1732519720813, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,33753,1732519720813, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5eb3d201e8c9 port=35771 startCode=1732519720972. As of locationSeqNum=12. 2024-11-25T07:28:57,468 DEBUG [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(1325): Online Regions={0197c07d98552bf4cd477bc9d94b6b62=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.} 2024-11-25T07:28:57,468 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,33753,1732519720813, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5eb3d201e8c9 port=35771 startCode=1732519720972. As of locationSeqNum=12. 2024-11-25T07:28:57,468 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,33753,1732519720813, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5eb3d201e8c9 port=35771 startCode=1732519720972. As of locationSeqNum=12. 2024-11-25T07:28:57,468 DEBUG [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(1351): Waiting on 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:57,470 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:57,470 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0197c07d98552bf4cd477bc9d94b6b62: Waiting for close lock at 1732519737466Running coprocessor pre-close hooks at 1732519737466Disabling compacts and flushes for region at 1732519737466Disabling writes for close at 1732519737467 (+1 ms)Writing region close event to WAL at 1732519737469 (+2 ms)Running coprocessor post-close hooks at 1732519737470 (+1 ms)Closed at 1732519737470 2024-11-25T07:28:57,470 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:57,527 INFO [regionserver/5eb3d201e8c9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T07:28:57,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 5eb3d201e8c9,35771,1732519720972 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:28:57,576 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 5eb3d201e8c9,35771,1732519720972 aborting 2024-11-25T07:28:57,576 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 5eb3d201e8c9,35771,1732519720972 aborting 2024-11-25T07:28:57,576 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=12 from cache 2024-11-25T07:28:57,668 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(976): stopping server 5eb3d201e8c9,35771,1732519720972; all regions closed. 2024-11-25T07:28:57,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741834_1010 (size=2142) 2024-11-25T07:28:57,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741834_1010 (size=2142) 2024-11-25T07:28:57,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741834_1010 (size=2142) 2024-11-25T07:28:57,673 DEBUG [RS:2;5eb3d201e8c9:35771 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:28:57,673 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T07:28:57,673 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T07:28:57,673 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.ChoreService(370): Chore service for: regionserver/5eb3d201e8c9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T07:28:57,673 INFO [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T07:28:57,673 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T07:28:57,673 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T07:28:57,673 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T07:28:57,673 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T07:28:57,674 INFO [RS:2;5eb3d201e8c9:35771 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35771 2024-11-25T07:28:57,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35771-0x1014e06dc560003, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5eb3d201e8c9,35771,1732519720972 2024-11-25T07:28:57,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T07:28:57,681 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T07:28:57,682 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5eb3d201e8c9,35771,1732519720972] 2024-11-25T07:28:57,684 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5eb3d201e8c9,35771,1732519720972 already deleted, retry=false 2024-11-25T07:28:57,684 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 5eb3d201e8c9,35771,1732519720972 on 5eb3d201e8c9,33903,1732519720118 2024-11-25T07:28:57,689 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 5eb3d201e8c9,35771,1732519720972, splitWal=true, meta=false 2024-11-25T07:28:57,692 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 5eb3d201e8c9,35771,1732519720972 (carryingMeta=false) 5eb3d201e8c9,35771,1732519720972/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@7eb0e845[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-25T07:28:57,692 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 5eb3d201e8c9,35771,1732519720972, splitWal=true, meta=false 2024-11-25T07:28:57,694 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 5eb3d201e8c9,35771,1732519720972 had 1 regions 2024-11-25T07:28:57,695 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 5eb3d201e8c9,35771,1732519720972, splitWal=true, meta=false, isMeta: false 2024-11-25T07:28:57,696 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972-splitting 2024-11-25T07:28:57,697 INFO [PEWorker-1 {}] master.SplitWALManager(105): 5eb3d201e8c9,35771,1732519720972 WAL count=1, meta=false 2024-11-25T07:28:57,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 5eb3d201e8c9%2C35771%2C1732519720972.1732519722293}] 2024-11-25T07:28:57,704 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=5eb3d201e8c9,33753,1732519720813 2024-11-25T07:28:57,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 5eb3d201e8c9%2C35771%2C1732519720972.1732519722293, worker=5eb3d201e8c9,33753,1732519720813}] 2024-11-25T07:28:57,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35771-0x1014e06dc560003, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T07:28:57,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35771-0x1014e06dc560003, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T07:28:57,784 INFO [RS:2;5eb3d201e8c9:35771 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T07:28:57,785 INFO [RS:2;5eb3d201e8c9:35771 {}] regionserver.HRegionServer(1031): Exiting; stopping=5eb3d201e8c9,35771,1732519720972; zookeeper connection closed. 2024-11-25T07:28:57,785 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1b48da09 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1b48da09 2024-11-25T07:28:57,787 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=18] 2024-11-25T07:28:57,788 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 5eb3d201e8c9:35771 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 5eb3d201e8c9/172.17.0.2:35771 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:28:57,789 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=18, error=java.net.ConnectException: Call to address=5eb3d201e8c9:35771 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 5eb3d201e8c9/172.17.0.2:35771 2024-11-25T07:28:57,789 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-25T07:28:57,789 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=18 from cache 2024-11-25T07:28:57,789 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 5eb3d201e8c9:35771 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 5eb3d201e8c9/172.17.0.2:35771 2024-11-25T07:28:57,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-25T07:28:57,885 INFO [RS_LOG_REPLAY_OPS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972-splitting/5eb3d201e8c9%2C35771%2C1732519720972.1732519722293, size=2.1 K (2142bytes) 2024-11-25T07:28:57,885 INFO [RS_LOG_REPLAY_OPS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972-splitting/5eb3d201e8c9%2C35771%2C1732519720972.1732519722293 2024-11-25T07:28:57,886 INFO [RS_LOG_REPLAY_OPS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972-splitting/5eb3d201e8c9%2C35771%2C1732519720972.1732519722293 after 1ms 2024-11-25T07:28:57,888 INFO [RS_LOG_REPLAY_OPS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972-splitting/5eb3d201e8c9%2C35771%2C1732519720972.1732519722293 took 3ms 2024-11-25T07:28:57,895 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 0197c07d98552bf4cd477bc9d94b6b62: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-25T07:28:57,895 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972-splitting/5eb3d201e8c9%2C35771%2C1732519720972.1732519722293 so closing down 2024-11-25T07:28:57,895 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T07:28:57,895 INFO [RS_LOG_REPLAY_OPS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T07:28:57,896 INFO [RS_LOG_REPLAY_OPS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972-splitting/5eb3d201e8c9%2C35771%2C1732519720972.1732519722293, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-11-25T07:28:57,896 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972-splitting/5eb3d201e8c9%2C35771%2C1732519720972.1732519722293, journal: Splitting hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972-splitting/5eb3d201e8c9%2C35771%2C1732519720972.1732519722293, size=2.1 K (2142bytes) at 1732519737885Finishing writing output for hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972-splitting/5eb3d201e8c9%2C35771%2C1732519720972.1732519722293 so closing down at 1732519737895 (+10 ms)3 split writer threads finished at 1732519737896 (+1 ms)Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972-splitting/5eb3d201e8c9%2C35771%2C1732519720972.1732519722293, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1732519737896 2024-11-25T07:28:57,896 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972-splitting/5eb3d201e8c9%2C35771%2C1732519720972.1732519722293 2024-11-25T07:28:57,897 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-25T07:28:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33903 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-25T07:28:57,903 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972-splitting/5eb3d201e8c9%2C35771%2C1732519720972.1732519722293 to hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/oldWALs 2024-11-25T07:28:57,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-25T07:28:57,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5eb3d201e8c9%2C35771%2C1732519720972.1732519722293, worker=5eb3d201e8c9,33753,1732519720813 in 197 msec 2024-11-25T07:28:57,906 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=5eb3d201e8c9,33753,1732519720813 2024-11-25T07:28:57,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-25T07:28:57,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 5eb3d201e8c9%2C35771%2C1732519720972.1732519722293, worker=5eb3d201e8c9,33753,1732519720813 in 209 msec 2024-11-25T07:28:57,910 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972-splitting dir is empty, no logs to split. 2024-11-25T07:28:57,911 INFO [PEWorker-1 {}] master.SplitWALManager(105): 5eb3d201e8c9,35771,1732519720972 WAL count=0, meta=false 2024-11-25T07:28:57,911 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 5eb3d201e8c9,35771,1732519720972 WAL splitting is done? wals=0, meta=false 2024-11-25T07:28:57,913 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 5eb3d201e8c9,35771,1732519720972 failed, ignore...File hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/WALs/5eb3d201e8c9,35771,1732519720972-splitting does not exist. 2024-11-25T07:28:57,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, ASSIGN}] 2024-11-25T07:28:57,916 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, ASSIGN 2024-11-25T07:28:57,917 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T07:28:58,068 DEBUG [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(204): Hosts are {5eb3d201e8c9=0} racks are {/default-rack=0} 2024-11-25T07:28:58,068 DEBUG [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T07:28:58,068 DEBUG [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T07:28:58,068 DEBUG [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T07:28:58,068 DEBUG [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T07:28:58,068 INFO [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T07:28:58,068 INFO [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T07:28:58,068 DEBUG [5eb3d201e8c9:33903 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T07:28:58,068 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=0197c07d98552bf4cd477bc9d94b6b62, regionState=OPENING, regionLocation=5eb3d201e8c9,33753,1732519720813 2024-11-25T07:28:58,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, ASSIGN because future has completed 2024-11-25T07:28:58,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,33753,1732519720813}] 2024-11-25T07:28:58,097 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=18] 2024-11-25T07:28:58,098 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 5eb3d201e8c9:35771 this server is in the failed servers list 2024-11-25T07:28:58,098 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=5eb3d201e8c9:35771 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 5eb3d201e8c9:35771 2024-11-25T07:28:58,098 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 5eb3d201e8c9:35771 2024-11-25T07:28:58,098 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,35771,1732519720972, seqNum=18 from cache 2024-11-25T07:28:58,228 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:58,228 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 0197c07d98552bf4cd477bc9d94b6b62, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:28:58,229 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:58,229 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:58,229 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:58,229 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:58,230 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:58,232 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0197c07d98552bf4cd477bc9d94b6b62 columnFamilyName cf1 2024-11-25T07:28:58,232 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:58,239 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf1/6316b588bbde43c8b43459e5d631fde2 2024-11-25T07:28:58,239 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(327): Store=0197c07d98552bf4cd477bc9d94b6b62/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:58,239 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:58,240 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0197c07d98552bf4cd477bc9d94b6b62 columnFamilyName cf2 2024-11-25T07:28:58,240 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:58,246 DEBUG [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/cf2/5b9a05600f0540289e73c40ade72246b 2024-11-25T07:28:58,247 INFO [StoreOpener-0197c07d98552bf4cd477bc9d94b6b62-1 {}] regionserver.HStore(327): Store=0197c07d98552bf4cd477bc9d94b6b62/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:58,247 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:58,248 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:58,249 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:58,250 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:58,250 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:58,250 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T07:28:58,252 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:58,252 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 0197c07d98552bf4cd477bc9d94b6b62; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61206474, jitterRate=-0.08795246481895447}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T07:28:58,253 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:28:58,253 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 0197c07d98552bf4cd477bc9d94b6b62: Running coprocessor pre-open hook at 1732519738229Writing region info on filesystem at 1732519738229Initializing all the Stores at 1732519738230 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519738230Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519738230Cleaning up temporary data from old regions at 1732519738250 (+20 ms)Running coprocessor post-open hooks at 1732519738253 (+3 ms)Region opened successfully at 1732519738253 2024-11-25T07:28:58,254 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., pid=17, masterSystemTime=1732519738224 2024-11-25T07:28:58,256 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:58,257 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:28:58,257 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=0197c07d98552bf4cd477bc9d94b6b62, regionState=OPEN, openSeqNum=18, regionLocation=5eb3d201e8c9,33753,1732519720813 2024-11-25T07:28:58,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,33753,1732519720813 because future has completed 2024-11-25T07:28:58,261 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33903 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=5eb3d201e8c9,33753,1732519720813, table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T07:28:58,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-25T07:28:58,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 0197c07d98552bf4cd477bc9d94b6b62, server=5eb3d201e8c9,33753,1732519720813 in 190 msec 2024-11-25T07:28:58,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-25T07:28:58,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=0197c07d98552bf4cd477bc9d94b6b62, ASSIGN in 349 msec 2024-11-25T07:28:58,266 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 5eb3d201e8c9,35771,1732519720972 after splitting done 2024-11-25T07:28:58,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 5eb3d201e8c9,35771,1732519720972, splitWal=true, meta=false in 581 msec 2024-11-25T07:28:58,607 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., hostname=5eb3d201e8c9,33753,1732519720813, seqNum=18] 2024-11-25T07:28:58,622 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=405 (was 414), OpenFileDescriptor=1045 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=121 (was 121), ProcessCount=11 (was 11), AvailableMemoryMB=8299 (was 8347) 2024-11-25T07:28:58,624 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1045 is superior to 1024 2024-11-25T07:28:58,637 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=405, OpenFileDescriptor=1045, MaxFileDescriptor=1048576, SystemLoadAverage=121, ProcessCount=11, AvailableMemoryMB=8299 2024-11-25T07:28:58,637 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1045 is superior to 1024 2024-11-25T07:28:58,654 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:58,656 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:58,656 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T07:28:58,659 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-01439074, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/hregion-01439074, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:58,673 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-01439074/hregion-01439074.1732519738659, exclude list is [], retry=0 2024-11-25T07:28:58,676 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:58,676 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:58,676 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:58,678 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-01439074/hregion-01439074.1732519738659 2024-11-25T07:28:58,679 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-25T07:28:58,679 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 11e25624527b4f9d778c7341422b5408, NAME => 'testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44261/hbase 2024-11-25T07:28:58,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741893_1071 (size=67) 2024-11-25T07:28:58,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741893_1071 (size=67) 2024-11-25T07:28:58,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741893_1071 (size=67) 2024-11-25T07:28:58,689 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:58,690 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,691 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11e25624527b4f9d778c7341422b5408 columnFamilyName a 2024-11-25T07:28:58,691 DEBUG [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:58,692 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(327): Store=11e25624527b4f9d778c7341422b5408/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:58,692 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,693 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11e25624527b4f9d778c7341422b5408 columnFamilyName b 2024-11-25T07:28:58,693 DEBUG [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:58,694 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(327): Store=11e25624527b4f9d778c7341422b5408/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:58,694 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,695 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11e25624527b4f9d778c7341422b5408 columnFamilyName c 2024-11-25T07:28:58,695 DEBUG [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:58,695 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(327): Store=11e25624527b4f9d778c7341422b5408/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:58,696 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,696 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,697 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,698 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,698 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,698 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:28:58,699 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,701 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:28:58,702 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 11e25624527b4f9d778c7341422b5408; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59989700, jitterRate=-0.10608381032943726}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:28:58,703 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 11e25624527b4f9d778c7341422b5408: Writing region info on filesystem at 1732519738689Initializing all the Stores at 1732519738690 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519738690Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519738690Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519738690Cleaning up temporary data from old regions at 1732519738698 (+8 ms)Region opened successfully at 1732519738703 (+5 ms) 2024-11-25T07:28:58,703 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 11e25624527b4f9d778c7341422b5408, disabling compactions & flushes 2024-11-25T07:28:58,703 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408. 2024-11-25T07:28:58,703 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408. 2024-11-25T07:28:58,703 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408. after waiting 0 ms 2024-11-25T07:28:58,703 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408. 2024-11-25T07:28:58,703 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408. 2024-11-25T07:28:58,704 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 11e25624527b4f9d778c7341422b5408: Waiting for close lock at 1732519738703Disabling compacts and flushes for region at 1732519738703Disabling writes for close at 1732519738703Writing region close event to WAL at 1732519738703Closed at 1732519738703 2024-11-25T07:28:58,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741892_1070 (size=93) 2024-11-25T07:28:58,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741892_1070 (size=93) 2024-11-25T07:28:58,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741892_1070 (size=93) 2024-11-25T07:28:58,708 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T07:28:58,708 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-01439074:(num 1732519738659) 2024-11-25T07:28:58,709 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:28:58,710 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:58,731 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738711, exclude list is [], retry=0 2024-11-25T07:28:58,734 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:58,734 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:58,734 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:58,736 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738711 2024-11-25T07:28:58,736 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-25T07:28:58,737 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 11e25624527b4f9d778c7341422b5408, NAME => 'testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:28:58,737 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:58,737 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,737 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,738 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,739 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11e25624527b4f9d778c7341422b5408 columnFamilyName a 2024-11-25T07:28:58,739 DEBUG [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:58,740 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(327): Store=11e25624527b4f9d778c7341422b5408/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:58,740 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,741 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11e25624527b4f9d778c7341422b5408 columnFamilyName b 2024-11-25T07:28:58,741 DEBUG [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:58,741 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(327): Store=11e25624527b4f9d778c7341422b5408/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:58,741 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,742 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11e25624527b4f9d778c7341422b5408 columnFamilyName c 2024-11-25T07:28:58,742 DEBUG [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:58,743 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(327): Store=11e25624527b4f9d778c7341422b5408/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:58,743 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,744 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,745 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,746 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,746 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,746 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:28:58,748 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,748 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 11e25624527b4f9d778c7341422b5408; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59732766, jitterRate=-0.10991242527961731}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:28:58,749 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 11e25624527b4f9d778c7341422b5408: Writing region info on filesystem at 1732519738737Initializing all the Stores at 1732519738738 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519738738Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519738738Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519738738Cleaning up temporary data from old regions at 1732519738746 (+8 ms)Region opened successfully at 1732519738749 (+3 ms) 2024-11-25T07:28:58,778 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 11e25624527b4f9d778c7341422b5408 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-25T07:28:58,802 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/.tmp/a/f4991057e16d4bc18f88be9f0cda5f69 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732519738749/Put/seqid=0 2024-11-25T07:28:58,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741895_1073 (size=5958) 2024-11-25T07:28:58,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741895_1073 (size=5958) 2024-11-25T07:28:58,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741895_1073 (size=5958) 2024-11-25T07:28:58,811 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/.tmp/a/f4991057e16d4bc18f88be9f0cda5f69 2024-11-25T07:28:58,839 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/.tmp/b/78bacccc3f354b19afcf290d42179b4f is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732519738758/Put/seqid=0 2024-11-25T07:28:58,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741896_1074 (size=5958) 2024-11-25T07:28:58,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741896_1074 (size=5958) 2024-11-25T07:28:58,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741896_1074 (size=5958) 2024-11-25T07:28:58,847 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/.tmp/b/78bacccc3f354b19afcf290d42179b4f 2024-11-25T07:28:58,875 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/.tmp/c/0b77b99d19d04e728ec460bf3b5fa166 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732519738765/Put/seqid=0 2024-11-25T07:28:58,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741897_1075 (size=5958) 2024-11-25T07:28:58,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741897_1075 (size=5958) 2024-11-25T07:28:58,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741897_1075 (size=5958) 2024-11-25T07:28:58,883 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/.tmp/c/0b77b99d19d04e728ec460bf3b5fa166 2024-11-25T07:28:58,889 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/.tmp/a/f4991057e16d4bc18f88be9f0cda5f69 as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/a/f4991057e16d4bc18f88be9f0cda5f69 2024-11-25T07:28:58,895 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/a/f4991057e16d4bc18f88be9f0cda5f69, entries=10, sequenceid=33, filesize=5.8 K 2024-11-25T07:28:58,895 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/.tmp/b/78bacccc3f354b19afcf290d42179b4f as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/b/78bacccc3f354b19afcf290d42179b4f 2024-11-25T07:28:58,901 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/b/78bacccc3f354b19afcf290d42179b4f, entries=10, sequenceid=33, filesize=5.8 K 2024-11-25T07:28:58,902 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/.tmp/c/0b77b99d19d04e728ec460bf3b5fa166 as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/c/0b77b99d19d04e728ec460bf3b5fa166 2024-11-25T07:28:58,907 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/c/0b77b99d19d04e728ec460bf3b5fa166, entries=10, sequenceid=33, filesize=5.8 K 2024-11-25T07:28:58,908 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 11e25624527b4f9d778c7341422b5408 in 130ms, sequenceid=33, compaction requested=false 2024-11-25T07:28:58,908 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 11e25624527b4f9d778c7341422b5408: 2024-11-25T07:28:58,908 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 11e25624527b4f9d778c7341422b5408, disabling compactions & flushes 2024-11-25T07:28:58,908 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408. 2024-11-25T07:28:58,908 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408. 2024-11-25T07:28:58,908 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408. after waiting 0 ms 2024-11-25T07:28:58,908 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408. 2024-11-25T07:28:58,910 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408. 2024-11-25T07:28:58,910 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 11e25624527b4f9d778c7341422b5408: Waiting for close lock at 1732519738908Disabling compacts and flushes for region at 1732519738908Disabling writes for close at 1732519738908Writing region close event to WAL at 1732519738910 (+2 ms)Closed at 1732519738910 2024-11-25T07:28:58,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741894_1072 (size=5937) 2024-11-25T07:28:58,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741894_1072 (size=5937) 2024-11-25T07:28:58,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741894_1072 (size=5937) 2024-11-25T07:28:58,919 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/b/78bacccc3f354b19afcf290d42179b4f to hdfs://localhost:44261/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/b/78bacccc3f354b19afcf290d42179b4f 2024-11-25T07:28:58,937 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44261/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738711, size=5.8 K (5937bytes) 2024-11-25T07:28:58,937 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44261/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738711 2024-11-25T07:28:58,937 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44261/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738711 after 0ms 2024-11-25T07:28:58,940 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44261/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738711 took 3ms 2024-11-25T07:28:58,942 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44261/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738711 so closing down 2024-11-25T07:28:58,942 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T07:28:58,944 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732519738711.temp 2024-11-25T07:28:58,945 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/recovered.edits/0000000000000000003-wal.1732519738711.temp 2024-11-25T07:28:58,945 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T07:28:58,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741898_1076 (size=5223) 2024-11-25T07:28:58,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741898_1076 (size=5223) 2024-11-25T07:28:58,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741898_1076 (size=5223) 2024-11-25T07:28:58,953 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/recovered.edits/0000000000000000003-wal.1732519738711.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-25T07:28:58,955 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/recovered.edits/0000000000000000003-wal.1732519738711.temp to hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/recovered.edits/0000000000000000032 2024-11-25T07:28:58,955 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:44261/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738711, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-11-25T07:28:58,955 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44261/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738711, journal: Splitting hdfs://localhost:44261/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738711, size=5.8 K (5937bytes) at 1732519738937Finishing writing output for hdfs://localhost:44261/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738711 so closing down at 1732519738942 (+5 ms)Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/recovered.edits/0000000000000000003-wal.1732519738711.temp at 1732519738945 (+3 ms)3 split writer threads finished at 1732519738945Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/recovered.edits/0000000000000000003-wal.1732519738711.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732519738954 (+9 ms)Rename recovered edits hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/recovered.edits/0000000000000000003-wal.1732519738711.temp to hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/recovered.edits/0000000000000000032 at 1732519738955 (+1 ms)Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:44261/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738711, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1732519738955 2024-11-25T07:28:58,957 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44261/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738711 to hdfs://localhost:44261/hbase/oldWALs/wal.1732519738711 2024-11-25T07:28:58,958 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/recovered.edits/0000000000000000032 2024-11-25T07:28:58,958 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:28:58,960 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:58,974 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738961, exclude list is [], retry=0 2024-11-25T07:28:58,977 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:58,977 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:58,978 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:58,980 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738961 2024-11-25T07:28:58,980 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:43429:43429)] 2024-11-25T07:28:58,980 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 11e25624527b4f9d778c7341422b5408, NAME => 'testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:28:58,980 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:58,980 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,980 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,982 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,983 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11e25624527b4f9d778c7341422b5408 columnFamilyName a 2024-11-25T07:28:58,983 DEBUG [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:58,988 DEBUG [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/a/f4991057e16d4bc18f88be9f0cda5f69 2024-11-25T07:28:58,988 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(327): Store=11e25624527b4f9d778c7341422b5408/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:58,988 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,990 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11e25624527b4f9d778c7341422b5408 columnFamilyName b 2024-11-25T07:28:58,990 DEBUG [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:58,990 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(327): Store=11e25624527b4f9d778c7341422b5408/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:58,990 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,991 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11e25624527b4f9d778c7341422b5408 columnFamilyName c 2024-11-25T07:28:58,991 DEBUG [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:58,997 DEBUG [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/c/0b77b99d19d04e728ec460bf3b5fa166 2024-11-25T07:28:58,997 INFO [StoreOpener-11e25624527b4f9d778c7341422b5408-1 {}] regionserver.HStore(327): Store=11e25624527b4f9d778c7341422b5408/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:58,997 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,997 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,999 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:58,999 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/recovered.edits/0000000000000000032 2024-11-25T07:28:59,002 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/recovered.edits/0000000000000000032 2024-11-25T07:28:59,002 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 11e25624527b4f9d778c7341422b5408 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-25T07:28:59,020 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/.tmp/b/bc7afce54f994c20b9627062eed5cf35 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732519738758/Put/seqid=0 2024-11-25T07:28:59,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741900_1078 (size=5958) 2024-11-25T07:28:59,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741900_1078 (size=5958) 2024-11-25T07:28:59,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741900_1078 (size=5958) 2024-11-25T07:28:59,029 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/.tmp/b/bc7afce54f994c20b9627062eed5cf35 2024-11-25T07:28:59,036 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/.tmp/b/bc7afce54f994c20b9627062eed5cf35 as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/b/bc7afce54f994c20b9627062eed5cf35 2024-11-25T07:28:59,042 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/b/bc7afce54f994c20b9627062eed5cf35, entries=10, sequenceid=32, filesize=5.8 K 2024-11-25T07:28:59,042 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 11e25624527b4f9d778c7341422b5408 in 40ms, sequenceid=32, compaction requested=false; wal=null 2024-11-25T07:28:59,043 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/recovered.edits/0000000000000000032 2024-11-25T07:28:59,044 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:59,044 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:59,045 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:28:59,046 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 11e25624527b4f9d778c7341422b5408 2024-11-25T07:28:59,049 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/11e25624527b4f9d778c7341422b5408/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-25T07:28:59,050 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 11e25624527b4f9d778c7341422b5408; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71985426, jitterRate=0.07266643643379211}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:28:59,051 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 11e25624527b4f9d778c7341422b5408: Writing region info on filesystem at 1732519738981Initializing all the Stores at 1732519738981Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519738981Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519738982 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519738982Obtaining lock to block concurrent updates at 1732519739002 (+20 ms)Preparing flush snapshotting stores in 11e25624527b4f9d778c7341422b5408 at 1732519739002Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732519739003 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1732519738654.11e25624527b4f9d778c7341422b5408. at 1732519739003Flushing 11e25624527b4f9d778c7341422b5408/b: creating writer at 1732519739003Flushing 11e25624527b4f9d778c7341422b5408/b: appending metadata at 1732519739019 (+16 ms)Flushing 11e25624527b4f9d778c7341422b5408/b: closing flushed file at 1732519739019Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@475cf3b6: reopening flushed file at 1732519739035 (+16 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 11e25624527b4f9d778c7341422b5408 in 40ms, sequenceid=32, compaction requested=false; wal=null at 1732519739042 (+7 ms)Cleaning up temporary data from old regions at 1732519739044 (+2 ms)Region opened successfully at 1732519739050 (+6 ms) 2024-11-25T07:28:59,074 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=414 (was 405) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-661403191-172.17.0.2-1732519717195:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:47700 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-661403191-172.17.0.2-1732519717195:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:53638 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:47552 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:53494 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:55958 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:55786 [Waiting for operation #38] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-661403191-172.17.0.2-1732519717195:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1121 (was 1045) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=121 (was 121), ProcessCount=11 (was 11), AvailableMemoryMB=8291 (was 8299) 2024-11-25T07:28:59,075 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1121 is superior to 1024 2024-11-25T07:28:59,091 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=414, OpenFileDescriptor=1121, MaxFileDescriptor=1048576, SystemLoadAverage=121, ProcessCount=11, AvailableMemoryMB=8290 2024-11-25T07:28:59,091 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1121 is superior to 1024 2024-11-25T07:28:59,115 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:59,118 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:28:59,118 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T07:28:59,122 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-70796029, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/hregion-70796029, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:59,136 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-70796029/hregion-70796029.1732519739122, exclude list is [], retry=0 2024-11-25T07:28:59,139 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:59,139 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:59,140 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:59,142 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-70796029/hregion-70796029.1732519739122 2024-11-25T07:28:59,142 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:37263:37263)] 2024-11-25T07:28:59,142 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6fc017d2fc400ba65c3d2bd8ba875977, NAME => 'testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44261/hbase 2024-11-25T07:28:59,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741902_1080 (size=68) 2024-11-25T07:28:59,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741902_1080 (size=68) 2024-11-25T07:28:59,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741902_1080 (size=68) 2024-11-25T07:28:59,155 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:59,157 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,158 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fc017d2fc400ba65c3d2bd8ba875977 columnFamilyName a 2024-11-25T07:28:59,158 DEBUG [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:59,159 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(327): Store=6fc017d2fc400ba65c3d2bd8ba875977/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:59,159 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,161 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fc017d2fc400ba65c3d2bd8ba875977 columnFamilyName b 2024-11-25T07:28:59,161 DEBUG [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:59,161 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(327): Store=6fc017d2fc400ba65c3d2bd8ba875977/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:59,161 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,163 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fc017d2fc400ba65c3d2bd8ba875977 columnFamilyName c 2024-11-25T07:28:59,163 DEBUG [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:59,163 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(327): Store=6fc017d2fc400ba65c3d2bd8ba875977/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:59,163 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,164 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,164 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,165 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,165 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,166 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:28:59,167 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,169 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:28:59,170 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6fc017d2fc400ba65c3d2bd8ba875977; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69553513, jitterRate=0.03642810881137848}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:28:59,171 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6fc017d2fc400ba65c3d2bd8ba875977: Writing region info on filesystem at 1732519739156Initializing all the Stores at 1732519739156Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519739156Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519739157 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519739157Cleaning up temporary data from old regions at 1732519739165 (+8 ms)Region opened successfully at 1732519739171 (+6 ms) 2024-11-25T07:28:59,171 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6fc017d2fc400ba65c3d2bd8ba875977, disabling compactions & flushes 2024-11-25T07:28:59,171 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977. 2024-11-25T07:28:59,171 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977. 2024-11-25T07:28:59,171 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977. after waiting 0 ms 2024-11-25T07:28:59,171 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977. 2024-11-25T07:28:59,171 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977. 2024-11-25T07:28:59,172 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6fc017d2fc400ba65c3d2bd8ba875977: Waiting for close lock at 1732519739171Disabling compacts and flushes for region at 1732519739171Disabling writes for close at 1732519739171Writing region close event to WAL at 1732519739171Closed at 1732519739171 2024-11-25T07:28:59,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741901_1079 (size=93) 2024-11-25T07:28:59,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741901_1079 (size=93) 2024-11-25T07:28:59,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741901_1079 (size=93) 2024-11-25T07:28:59,181 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T07:28:59,181 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-70796029:(num 1732519739122) 2024-11-25T07:28:59,181 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:28:59,183 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:28:59,197 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519739184, exclude list is [], retry=0 2024-11-25T07:28:59,200 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:28:59,200 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:28:59,200 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:28:59,202 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519739184 2024-11-25T07:28:59,202 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-25T07:28:59,257 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6fc017d2fc400ba65c3d2bd8ba875977, NAME => 'testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:28:59,259 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,260 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:28:59,260 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,260 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,262 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,263 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fc017d2fc400ba65c3d2bd8ba875977 columnFamilyName a 2024-11-25T07:28:59,263 DEBUG [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:59,263 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(327): Store=6fc017d2fc400ba65c3d2bd8ba875977/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:59,264 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,264 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fc017d2fc400ba65c3d2bd8ba875977 columnFamilyName b 2024-11-25T07:28:59,265 DEBUG [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:59,265 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(327): Store=6fc017d2fc400ba65c3d2bd8ba875977/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:59,265 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,266 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fc017d2fc400ba65c3d2bd8ba875977 columnFamilyName c 2024-11-25T07:28:59,266 DEBUG [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:28:59,266 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(327): Store=6fc017d2fc400ba65c3d2bd8ba875977/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:28:59,267 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,267 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,269 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,270 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,270 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,270 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:28:59,272 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,273 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6fc017d2fc400ba65c3d2bd8ba875977; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69434795, jitterRate=0.03465907275676727}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:28:59,273 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:28:59,274 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6fc017d2fc400ba65c3d2bd8ba875977: Running coprocessor pre-open hook at 1732519739260Writing region info on filesystem at 1732519739260Initializing all the Stores at 1732519739261 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519739261Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519739261Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519739261Cleaning up temporary data from old regions at 1732519739270 (+9 ms)Running coprocessor post-open hooks at 1732519739273 (+3 ms)Region opened successfully at 1732519739273 2024-11-25T07:28:59,286 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6fc017d2fc400ba65c3d2bd8ba875977 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-25T07:28:59,288 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6fc017d2fc400ba65c3d2bd8ba875977/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:00,289 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6fc017d2fc400ba65c3d2bd8ba875977/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:00,529 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-25T07:29:00,530 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-25T07:29:00,531 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-25T07:29:00,531 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-25T07:29:00,532 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-25T07:29:00,532 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-25T07:29:00,532 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-25T07:29:00,532 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-25T07:29:00,533 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-25T07:29:00,533 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-25T07:29:01,183 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T07:29:01,289 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6fc017d2fc400ba65c3d2bd8ba875977/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:02,290 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6fc017d2fc400ba65c3d2bd8ba875977/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:03,291 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6fc017d2fc400ba65c3d2bd8ba875977/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:04,291 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6fc017d2fc400ba65c3d2bd8ba875977/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:05,292 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6fc017d2fc400ba65c3d2bd8ba875977/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:06,293 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6fc017d2fc400ba65c3d2bd8ba875977/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:07,293 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6fc017d2fc400ba65c3d2bd8ba875977/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:08,294 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6fc017d2fc400ba65c3d2bd8ba875977/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:08,295 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6fc017d2fc400ba65c3d2bd8ba875977: 2024-11-25T07:29:08,296 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977. 2024-11-25T07:29:08,311 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6fc017d2fc400ba65c3d2bd8ba875977: 2024-11-25T07:29:08,311 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-25T07:29:08,311 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6fc017d2fc400ba65c3d2bd8ba875977, disabling compactions & flushes 2024-11-25T07:29:08,311 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977. 2024-11-25T07:29:08,311 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977. 2024-11-25T07:29:08,311 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977. after waiting 0 ms 2024-11-25T07:29:08,311 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977. 2024-11-25T07:29:08,312 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977. 2024-11-25T07:29:08,312 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977. 2024-11-25T07:29:08,312 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6fc017d2fc400ba65c3d2bd8ba875977: Waiting for close lock at 1732519748311Running coprocessor pre-close hooks at 1732519748311Disabling compacts and flushes for region at 1732519748311Disabling writes for close at 1732519748311Writing region close event to WAL at 1732519748312 (+1 ms)Running coprocessor post-close hooks at 1732519748312Closed at 1732519748312 2024-11-25T07:29:08,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741903_1081 (size=3941) 2024-11-25T07:29:08,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741903_1081 (size=3941) 2024-11-25T07:29:08,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741903_1081 (size=3941) 2024-11-25T07:29:08,332 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44261/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519739184, size=3.8 K (3941bytes) 2024-11-25T07:29:08,332 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44261/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519739184 2024-11-25T07:29:08,332 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44261/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519739184 after 0ms 2024-11-25T07:29:08,335 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44261/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519739184 took 3ms 2024-11-25T07:29:08,337 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44261/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519739184 so closing down 2024-11-25T07:29:08,337 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T07:29:08,339 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732519739184.temp 2024-11-25T07:29:08,340 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/recovered.edits/0000000000000000004-wal.1732519739184.temp 2024-11-25T07:29:08,340 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T07:29:08,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741904_1082 (size=2983) 2024-11-25T07:29:08,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741904_1082 (size=2983) 2024-11-25T07:29:08,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741904_1082 (size=2983) 2024-11-25T07:29:08,354 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/recovered.edits/0000000000000000004-wal.1732519739184.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-25T07:29:08,355 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/recovered.edits/0000000000000000004-wal.1732519739184.temp to hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/recovered.edits/0000000000000000026 2024-11-25T07:29:08,355 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 19 ms; skipped=3; WAL=hdfs://localhost:44261/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519739184, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-11-25T07:29:08,355 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44261/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519739184, journal: Splitting hdfs://localhost:44261/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519739184, size=3.8 K (3941bytes) at 1732519748332Finishing writing output for hdfs://localhost:44261/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519739184 so closing down at 1732519748337 (+5 ms)Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/recovered.edits/0000000000000000004-wal.1732519739184.temp at 1732519748340 (+3 ms)3 split writer threads finished at 1732519748340Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/recovered.edits/0000000000000000004-wal.1732519739184.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732519748354 (+14 ms)Rename recovered edits hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/recovered.edits/0000000000000000004-wal.1732519739184.temp to hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/recovered.edits/0000000000000000026 at 1732519748355 (+1 ms)Processed 23 edits across 1 Regions in 19 ms; skipped=3; WAL=hdfs://localhost:44261/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519739184, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1732519748355 2024-11-25T07:29:08,357 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44261/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519739184 to hdfs://localhost:44261/hbase/oldWALs/wal.1732519739184 2024-11-25T07:29:08,358 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/recovered.edits/0000000000000000026 2024-11-25T07:29:08,358 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:29:08,360 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:29:08,374 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519748360, exclude list is [], retry=0 2024-11-25T07:29:08,376 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:29:08,377 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:29:08,377 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:29:08,378 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519748360 2024-11-25T07:29:08,379 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-25T07:29:08,379 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6fc017d2fc400ba65c3d2bd8ba875977, NAME => 'testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:29:08,380 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:29:08,380 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:29:08,380 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:29:08,380 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:29:08,382 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:29:08,383 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fc017d2fc400ba65c3d2bd8ba875977 columnFamilyName a 2024-11-25T07:29:08,383 DEBUG [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:08,383 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(327): Store=6fc017d2fc400ba65c3d2bd8ba875977/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:08,384 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:29:08,384 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fc017d2fc400ba65c3d2bd8ba875977 columnFamilyName b 2024-11-25T07:29:08,384 DEBUG [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:08,385 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(327): Store=6fc017d2fc400ba65c3d2bd8ba875977/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:08,385 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:29:08,386 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fc017d2fc400ba65c3d2bd8ba875977 columnFamilyName c 2024-11-25T07:29:08,386 DEBUG [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:08,386 INFO [StoreOpener-6fc017d2fc400ba65c3d2bd8ba875977-1 {}] regionserver.HStore(327): Store=6fc017d2fc400ba65c3d2bd8ba875977/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:08,386 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:29:08,387 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:29:08,388 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:29:08,389 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/recovered.edits/0000000000000000026 2024-11-25T07:29:08,392 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/recovered.edits/0000000000000000026 2024-11-25T07:29:08,392 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6fc017d2fc400ba65c3d2bd8ba875977 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-25T07:29:08,407 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/.tmp/a/b8abcc4808f94a2f823e88555921775e is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732519748301/Put/seqid=0 2024-11-25T07:29:08,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741906_1084 (size=5523) 2024-11-25T07:29:08,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741906_1084 (size=5523) 2024-11-25T07:29:08,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741906_1084 (size=5523) 2024-11-25T07:29:08,415 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/.tmp/a/b8abcc4808f94a2f823e88555921775e 2024-11-25T07:29:08,436 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/.tmp/b/50d5d6fdfb6a4358925ce0ef0d27bbbd is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732519748296/Put/seqid=0 2024-11-25T07:29:08,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741907_1085 (size=5524) 2024-11-25T07:29:08,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741907_1085 (size=5524) 2024-11-25T07:29:08,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741907_1085 (size=5524) 2024-11-25T07:29:08,445 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/.tmp/b/50d5d6fdfb6a4358925ce0ef0d27bbbd 2024-11-25T07:29:08,465 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/.tmp/c/9ab2ee2b72a54bdcba5dd2f80c98815d is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732519748298/Put/seqid=0 2024-11-25T07:29:08,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741908_1086 (size=5457) 2024-11-25T07:29:08,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741908_1086 (size=5457) 2024-11-25T07:29:08,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741908_1086 (size=5457) 2024-11-25T07:29:08,473 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/.tmp/c/9ab2ee2b72a54bdcba5dd2f80c98815d 2024-11-25T07:29:08,479 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/.tmp/a/b8abcc4808f94a2f823e88555921775e as hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/a/b8abcc4808f94a2f823e88555921775e 2024-11-25T07:29:08,485 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/a/b8abcc4808f94a2f823e88555921775e, entries=7, sequenceid=26, filesize=5.4 K 2024-11-25T07:29:08,486 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/.tmp/b/50d5d6fdfb6a4358925ce0ef0d27bbbd as hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/b/50d5d6fdfb6a4358925ce0ef0d27bbbd 2024-11-25T07:29:08,492 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/b/50d5d6fdfb6a4358925ce0ef0d27bbbd, entries=7, sequenceid=26, filesize=5.4 K 2024-11-25T07:29:08,493 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/.tmp/c/9ab2ee2b72a54bdcba5dd2f80c98815d as hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/c/9ab2ee2b72a54bdcba5dd2f80c98815d 2024-11-25T07:29:08,498 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/c/9ab2ee2b72a54bdcba5dd2f80c98815d, entries=6, sequenceid=26, filesize=5.3 K 2024-11-25T07:29:08,499 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 6fc017d2fc400ba65c3d2bd8ba875977 in 107ms, sequenceid=26, compaction requested=false; wal=null 2024-11-25T07:29:08,499 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/recovered.edits/0000000000000000026 2024-11-25T07:29:08,501 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:29:08,501 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:29:08,501 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:29:08,503 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:29:08,505 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testReplayEditsAfterAbortingFlush/6fc017d2fc400ba65c3d2bd8ba875977/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-25T07:29:08,506 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6fc017d2fc400ba65c3d2bd8ba875977; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70522113, jitterRate=0.05086137354373932}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:29:08,506 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6fc017d2fc400ba65c3d2bd8ba875977 2024-11-25T07:29:08,507 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6fc017d2fc400ba65c3d2bd8ba875977: Running coprocessor pre-open hook at 1732519748380Writing region info on filesystem at 1732519748380Initializing all the Stores at 1732519748381 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519748381Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519748381Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519748381Obtaining lock to block concurrent updates at 1732519748392 (+11 ms)Preparing flush snapshotting stores in 6fc017d2fc400ba65c3d2bd8ba875977 at 1732519748392Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732519748393 (+1 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1732519739116.6fc017d2fc400ba65c3d2bd8ba875977. at 1732519748393Flushing 6fc017d2fc400ba65c3d2bd8ba875977/a: creating writer at 1732519748393Flushing 6fc017d2fc400ba65c3d2bd8ba875977/a: appending metadata at 1732519748407 (+14 ms)Flushing 6fc017d2fc400ba65c3d2bd8ba875977/a: closing flushed file at 1732519748407Flushing 6fc017d2fc400ba65c3d2bd8ba875977/b: creating writer at 1732519748421 (+14 ms)Flushing 6fc017d2fc400ba65c3d2bd8ba875977/b: appending metadata at 1732519748436 (+15 ms)Flushing 6fc017d2fc400ba65c3d2bd8ba875977/b: closing flushed file at 1732519748436Flushing 6fc017d2fc400ba65c3d2bd8ba875977/c: creating writer at 1732519748450 (+14 ms)Flushing 6fc017d2fc400ba65c3d2bd8ba875977/c: appending metadata at 1732519748465 (+15 ms)Flushing 6fc017d2fc400ba65c3d2bd8ba875977/c: closing flushed file at 1732519748465Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5704aa3e: reopening flushed file at 1732519748478 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@604eacf: reopening flushed file at 1732519748485 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15c6f9cf: reopening flushed file at 1732519748492 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 6fc017d2fc400ba65c3d2bd8ba875977 in 107ms, sequenceid=26, compaction requested=false; wal=null at 1732519748499 (+7 ms)Cleaning up temporary data from old regions at 1732519748501 (+2 ms)Running coprocessor post-open hooks at 1732519748506 (+5 ms)Region opened successfully at 1732519748507 (+1 ms) 2024-11-25T07:29:08,534 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=413 (was 414), OpenFileDescriptor=1179 (was 1121) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=117 (was 121), ProcessCount=11 (was 11), AvailableMemoryMB=8231 (was 8290) 2024-11-25T07:29:08,534 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1179 is superior to 1024 2024-11-25T07:29:08,547 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=413, OpenFileDescriptor=1179, MaxFileDescriptor=1048576, SystemLoadAverage=117, ProcessCount=11, AvailableMemoryMB=8230 2024-11-25T07:29:08,547 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1179 is superior to 1024 2024-11-25T07:29:08,563 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:29:08,565 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:29:08,566 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T07:29:08,568 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-91271062, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/hregion-91271062, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:29:08,582 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-91271062/hregion-91271062.1732519748569, exclude list is [], retry=0 2024-11-25T07:29:08,585 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:29:08,585 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:29:08,585 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:29:08,587 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-91271062/hregion-91271062.1732519748569 2024-11-25T07:29:08,588 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:43429:43429)] 2024-11-25T07:29:08,588 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d0e0092df20d2e84712ce86ae6633e1c, NAME => 'testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44261/hbase 2024-11-25T07:29:08,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741910_1088 (size=61) 2024-11-25T07:29:08,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741910_1088 (size=61) 2024-11-25T07:29:08,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741910_1088 (size=61) 2024-11-25T07:29:08,597 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:29:08,599 INFO [StoreOpener-d0e0092df20d2e84712ce86ae6633e1c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:08,600 INFO [StoreOpener-d0e0092df20d2e84712ce86ae6633e1c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0e0092df20d2e84712ce86ae6633e1c columnFamilyName a 2024-11-25T07:29:08,600 DEBUG [StoreOpener-d0e0092df20d2e84712ce86ae6633e1c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:08,601 INFO [StoreOpener-d0e0092df20d2e84712ce86ae6633e1c-1 {}] regionserver.HStore(327): Store=d0e0092df20d2e84712ce86ae6633e1c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:08,601 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:08,601 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:08,602 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:08,602 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:08,602 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:08,604 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:08,606 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:29:08,606 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d0e0092df20d2e84712ce86ae6633e1c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66680889, jitterRate=-0.006377324461936951}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:29:08,607 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d0e0092df20d2e84712ce86ae6633e1c: Writing region info on filesystem at 1732519748598Initializing all the Stores at 1732519748598Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519748598Cleaning up temporary data from old regions at 1732519748602 (+4 ms)Region opened successfully at 1732519748607 (+5 ms) 2024-11-25T07:29:08,607 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d0e0092df20d2e84712ce86ae6633e1c, disabling compactions & flushes 2024-11-25T07:29:08,607 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c. 2024-11-25T07:29:08,607 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c. 2024-11-25T07:29:08,607 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c. after waiting 0 ms 2024-11-25T07:29:08,607 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c. 2024-11-25T07:29:08,607 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c. 2024-11-25T07:29:08,607 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d0e0092df20d2e84712ce86ae6633e1c: Waiting for close lock at 1732519748607Disabling compacts and flushes for region at 1732519748607Disabling writes for close at 1732519748607Writing region close event to WAL at 1732519748607Closed at 1732519748607 2024-11-25T07:29:08,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741909_1087 (size=93) 2024-11-25T07:29:08,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741909_1087 (size=93) 2024-11-25T07:29:08,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741909_1087 (size=93) 2024-11-25T07:29:08,613 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T07:29:08,613 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-91271062:(num 1732519748569) 2024-11-25T07:29:08,613 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:29:08,615 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:29:08,636 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519748615, exclude list is [], retry=0 2024-11-25T07:29:08,639 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:29:08,639 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:29:08,640 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:29:08,641 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519748615 2024-11-25T07:29:08,641 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:43429:43429)] 2024-11-25T07:29:08,642 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d0e0092df20d2e84712ce86ae6633e1c, NAME => 'testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:29:08,642 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:29:08,642 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:08,642 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:08,643 INFO [StoreOpener-d0e0092df20d2e84712ce86ae6633e1c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:08,644 INFO [StoreOpener-d0e0092df20d2e84712ce86ae6633e1c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0e0092df20d2e84712ce86ae6633e1c columnFamilyName a 2024-11-25T07:29:08,644 DEBUG [StoreOpener-d0e0092df20d2e84712ce86ae6633e1c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:08,645 INFO [StoreOpener-d0e0092df20d2e84712ce86ae6633e1c-1 {}] regionserver.HStore(327): Store=d0e0092df20d2e84712ce86ae6633e1c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:08,645 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:08,645 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:08,647 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:08,647 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:08,647 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:08,649 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:08,650 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d0e0092df20d2e84712ce86ae6633e1c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74773181, jitterRate=0.11420722305774689}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:29:08,650 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d0e0092df20d2e84712ce86ae6633e1c: Writing region info on filesystem at 1732519748642Initializing all the Stores at 1732519748643 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519748643Cleaning up temporary data from old regions at 1732519748647 (+4 ms)Region opened successfully at 1732519748650 (+3 ms) 2024-11-25T07:29:08,660 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d0e0092df20d2e84712ce86ae6633e1c, disabling compactions & flushes 2024-11-25T07:29:08,660 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c. 2024-11-25T07:29:08,660 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c. 2024-11-25T07:29:08,660 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c. after waiting 0 ms 2024-11-25T07:29:08,660 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c. 2024-11-25T07:29:08,661 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c. 2024-11-25T07:29:08,661 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c. 2024-11-25T07:29:08,661 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d0e0092df20d2e84712ce86ae6633e1c: Waiting for close lock at 1732519748660Disabling compacts and flushes for region at 1732519748660Disabling writes for close at 1732519748660Writing region close event to WAL at 1732519748661 (+1 ms)Closed at 1732519748661 2024-11-25T07:29:08,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741911_1089 (size=1623) 2024-11-25T07:29:08,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741911_1089 (size=1623) 2024-11-25T07:29:08,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741911_1089 (size=1623) 2024-11-25T07:29:08,680 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44261/hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519748615, size=1.6 K (1623bytes) 2024-11-25T07:29:08,680 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44261/hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519748615 2024-11-25T07:29:08,680 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44261/hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519748615 after 0ms 2024-11-25T07:29:08,682 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44261/hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519748615 took 2ms 2024-11-25T07:29:08,684 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44261/hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519748615 so closing down 2024-11-25T07:29:08,684 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T07:29:08,685 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732519748615.temp 2024-11-25T07:29:08,686 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/recovered.edits/0000000000000000003-wal.1732519748615.temp 2024-11-25T07:29:08,686 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T07:29:08,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741912_1090 (size=1623) 2024-11-25T07:29:08,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741912_1090 (size=1623) 2024-11-25T07:29:08,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741912_1090 (size=1623) 2024-11-25T07:29:08,693 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/recovered.edits/0000000000000000003-wal.1732519748615.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-25T07:29:08,695 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/recovered.edits/0000000000000000003-wal.1732519748615.temp to hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/recovered.edits/0000000000000000012 2024-11-25T07:29:08,695 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519748615, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-11-25T07:29:08,695 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44261/hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519748615, journal: Splitting hdfs://localhost:44261/hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519748615, size=1.6 K (1623bytes) at 1732519748680Finishing writing output for hdfs://localhost:44261/hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519748615 so closing down at 1732519748684 (+4 ms)Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/recovered.edits/0000000000000000003-wal.1732519748615.temp at 1732519748686 (+2 ms)3 split writer threads finished at 1732519748686Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/recovered.edits/0000000000000000003-wal.1732519748615.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732519748693 (+7 ms)Rename recovered edits hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/recovered.edits/0000000000000000003-wal.1732519748615.temp to hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/recovered.edits/0000000000000000012 at 1732519748695 (+2 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519748615, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1732519748695 2024-11-25T07:29:08,697 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44261/hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519748615 to hdfs://localhost:44261/hbase/oldWALs/wal.1732519748615 2024-11-25T07:29:08,697 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/recovered.edits/0000000000000000012 2024-11-25T07:29:09,047 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:29:09,050 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:29:09,066 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519749050, exclude list is [], retry=0 2024-11-25T07:29:09,068 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:29:09,069 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:29:09,069 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:29:09,071 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519749050 2024-11-25T07:29:09,071 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:37263:37263)] 2024-11-25T07:29:09,071 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d0e0092df20d2e84712ce86ae6633e1c, NAME => 'testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:29:09,071 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:29:09,072 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,072 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,074 INFO [StoreOpener-d0e0092df20d2e84712ce86ae6633e1c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,076 INFO [StoreOpener-d0e0092df20d2e84712ce86ae6633e1c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0e0092df20d2e84712ce86ae6633e1c columnFamilyName a 2024-11-25T07:29:09,076 DEBUG [StoreOpener-d0e0092df20d2e84712ce86ae6633e1c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:09,077 INFO [StoreOpener-d0e0092df20d2e84712ce86ae6633e1c-1 {}] regionserver.HStore(327): Store=d0e0092df20d2e84712ce86ae6633e1c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:09,077 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,077 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,079 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,079 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/recovered.edits/0000000000000000012 2024-11-25T07:29:09,083 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/recovered.edits/0000000000000000012 2024-11-25T07:29:09,083 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d0e0092df20d2e84712ce86ae6633e1c 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-25T07:29:09,109 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/.tmp/a/b5eb6736877e4d608885f413a1a80621 is 79, key is testDatalossWhenInputError/a:x0/1732519748650/Put/seqid=0 2024-11-25T07:29:09,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741914_1092 (size=5808) 2024-11-25T07:29:09,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741914_1092 (size=5808) 2024-11-25T07:29:09,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741914_1092 (size=5808) 2024-11-25T07:29:09,120 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/.tmp/a/b5eb6736877e4d608885f413a1a80621 2024-11-25T07:29:09,132 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/.tmp/a/b5eb6736877e4d608885f413a1a80621 as hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/a/b5eb6736877e4d608885f413a1a80621 2024-11-25T07:29:09,143 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/a/b5eb6736877e4d608885f413a1a80621, entries=10, sequenceid=12, filesize=5.7 K 2024-11-25T07:29:09,144 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for d0e0092df20d2e84712ce86ae6633e1c in 60ms, sequenceid=12, compaction requested=false; wal=null 2024-11-25T07:29:09,144 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/recovered.edits/0000000000000000012 2024-11-25T07:29:09,145 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,145 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,148 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,150 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-25T07:29:09,151 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d0e0092df20d2e84712ce86ae6633e1c; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62248556, jitterRate=-0.07242423295974731}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:29:09,152 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d0e0092df20d2e84712ce86ae6633e1c: Writing region info on filesystem at 1732519749072Initializing all the Stores at 1732519749074 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519749074Obtaining lock to block concurrent updates at 1732519749083 (+9 ms)Preparing flush snapshotting stores in d0e0092df20d2e84712ce86ae6633e1c at 1732519749083Finished memstore snapshotting testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732519749083Flushing stores of testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c. at 1732519749083Flushing d0e0092df20d2e84712ce86ae6633e1c/a: creating writer at 1732519749083Flushing d0e0092df20d2e84712ce86ae6633e1c/a: appending metadata at 1732519749108 (+25 ms)Flushing d0e0092df20d2e84712ce86ae6633e1c/a: closing flushed file at 1732519749108Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c9c33bd: reopening flushed file at 1732519749130 (+22 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for d0e0092df20d2e84712ce86ae6633e1c in 60ms, sequenceid=12, compaction requested=false; wal=null at 1732519749144 (+14 ms)Cleaning up temporary data from old regions at 1732519749146 (+2 ms)Region opened successfully at 1732519749152 (+6 ms) 2024-11-25T07:29:09,154 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d0e0092df20d2e84712ce86ae6633e1c, NAME => 'testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:29:09,155 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732519748564.d0e0092df20d2e84712ce86ae6633e1c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:29:09,155 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,155 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,156 INFO [StoreOpener-d0e0092df20d2e84712ce86ae6633e1c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,157 INFO [StoreOpener-d0e0092df20d2e84712ce86ae6633e1c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0e0092df20d2e84712ce86ae6633e1c columnFamilyName a 2024-11-25T07:29:09,157 DEBUG [StoreOpener-d0e0092df20d2e84712ce86ae6633e1c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:09,162 DEBUG [StoreOpener-d0e0092df20d2e84712ce86ae6633e1c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/a/b5eb6736877e4d608885f413a1a80621 2024-11-25T07:29:09,162 INFO [StoreOpener-d0e0092df20d2e84712ce86ae6633e1c-1 {}] regionserver.HStore(327): Store=d0e0092df20d2e84712ce86ae6633e1c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:09,162 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,163 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,164 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,165 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,165 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,167 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d0e0092df20d2e84712ce86ae6633e1c 2024-11-25T07:29:09,169 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testDatalossWhenInputError/d0e0092df20d2e84712ce86ae6633e1c/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-25T07:29:09,170 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d0e0092df20d2e84712ce86ae6633e1c; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71124779, jitterRate=0.05984179675579071}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:29:09,171 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d0e0092df20d2e84712ce86ae6633e1c: Writing region info on filesystem at 1732519749155Initializing all the Stores at 1732519749156 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519749156Cleaning up temporary data from old regions at 1732519749165 (+9 ms)Region opened successfully at 1732519749171 (+6 ms) 2024-11-25T07:29:09,195 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=423 (was 413) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:58560 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:40760 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:58646 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-661403191-172.17.0.2-1732519717195:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-661403191-172.17.0.2-1732519717195:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-661403191-172.17.0.2-1732519717195:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:40838 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:38184 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:38246 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1261 (was 1179) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=117 (was 117), ProcessCount=11 (was 11), AvailableMemoryMB=8219 (was 8230) 2024-11-25T07:29:09,195 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1261 is superior to 1024 2024-11-25T07:29:09,210 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=423, OpenFileDescriptor=1261, MaxFileDescriptor=1048576, SystemLoadAverage=117, ProcessCount=11, AvailableMemoryMB=8218 2024-11-25T07:29:09,210 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1261 is superior to 1024 2024-11-25T07:29:09,233 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:29:09,235 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:29:09,235 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T07:29:09,238 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-38471862, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/hregion-38471862, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:29:09,252 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-38471862/hregion-38471862.1732519749238, exclude list is [], retry=0 2024-11-25T07:29:09,255 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:29:09,255 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:29:09,255 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:29:09,260 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-38471862/hregion-38471862.1732519749238 2024-11-25T07:29:09,261 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:43429:43429)] 2024-11-25T07:29:09,261 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 273fc93dcce824ae865261ee6fdcfafd, NAME => 'testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44261/hbase 2024-11-25T07:29:09,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741916_1094 (size=63) 2024-11-25T07:29:09,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741916_1094 (size=63) 2024-11-25T07:29:09,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741916_1094 (size=63) 2024-11-25T07:29:09,272 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:29:09,275 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T07:29:09,280 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,282 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 273fc93dcce824ae865261ee6fdcfafd columnFamilyName a 2024-11-25T07:29:09,282 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:09,282 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(327): Store=273fc93dcce824ae865261ee6fdcfafd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:09,283 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,284 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 273fc93dcce824ae865261ee6fdcfafd columnFamilyName b 2024-11-25T07:29:09,284 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:09,285 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(327): Store=273fc93dcce824ae865261ee6fdcfafd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:09,285 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,286 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 273fc93dcce824ae865261ee6fdcfafd columnFamilyName c 2024-11-25T07:29:09,286 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:09,287 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(327): Store=273fc93dcce824ae865261ee6fdcfafd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:09,287 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,288 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,289 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,291 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,291 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,291 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:29:09,293 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,295 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:29:09,295 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 273fc93dcce824ae865261ee6fdcfafd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67134674, jitterRate=3.84598970413208E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:29:09,296 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 273fc93dcce824ae865261ee6fdcfafd: Writing region info on filesystem at 1732519749272Initializing all the Stores at 1732519749279 (+7 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519749279Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519749280 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519749280Cleaning up temporary data from old regions at 1732519749291 (+11 ms)Region opened successfully at 1732519749296 (+5 ms) 2024-11-25T07:29:09,296 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 273fc93dcce824ae865261ee6fdcfafd, disabling compactions & flushes 2024-11-25T07:29:09,296 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd. 2024-11-25T07:29:09,296 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd. 2024-11-25T07:29:09,296 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd. after waiting 0 ms 2024-11-25T07:29:09,296 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd. 2024-11-25T07:29:09,296 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd. 2024-11-25T07:29:09,297 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 273fc93dcce824ae865261ee6fdcfafd: Waiting for close lock at 1732519749296Disabling compacts and flushes for region at 1732519749296Disabling writes for close at 1732519749296Writing region close event to WAL at 1732519749296Closed at 1732519749296 2024-11-25T07:29:09,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741915_1093 (size=93) 2024-11-25T07:29:09,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741915_1093 (size=93) 2024-11-25T07:29:09,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741915_1093 (size=93) 2024-11-25T07:29:09,302 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T07:29:09,302 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-38471862:(num 1732519749238) 2024-11-25T07:29:09,302 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:29:09,304 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:29:09,318 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304, exclude list is [], retry=0 2024-11-25T07:29:09,320 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:29:09,321 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:29:09,321 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:29:09,323 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304 2024-11-25T07:29:09,323 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-25T07:29:09,323 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 273fc93dcce824ae865261ee6fdcfafd, NAME => 'testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:29:09,323 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:29:09,324 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,324 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,326 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,327 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 273fc93dcce824ae865261ee6fdcfafd columnFamilyName a 2024-11-25T07:29:09,327 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:09,327 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(327): Store=273fc93dcce824ae865261ee6fdcfafd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:09,328 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,328 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 273fc93dcce824ae865261ee6fdcfafd columnFamilyName b 2024-11-25T07:29:09,329 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:09,329 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(327): Store=273fc93dcce824ae865261ee6fdcfafd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:09,329 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,330 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 273fc93dcce824ae865261ee6fdcfafd columnFamilyName c 2024-11-25T07:29:09,330 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:09,330 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(327): Store=273fc93dcce824ae865261ee6fdcfafd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:09,330 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,331 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,332 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,333 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,333 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,334 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:29:09,335 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:09,336 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 273fc93dcce824ae865261ee6fdcfafd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73972483, jitterRate=0.10227589309215546}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:29:09,337 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 273fc93dcce824ae865261ee6fdcfafd: Writing region info on filesystem at 1732519749324Initializing all the Stores at 1732519749325 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519749325Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519749326 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519749326Cleaning up temporary data from old regions at 1732519749333 (+7 ms)Region opened successfully at 1732519749337 (+4 ms) 2024-11-25T07:29:09,344 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732519749343/Put/seqid=0 2024-11-25T07:29:09,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741918_1096 (size=4875) 2024-11-25T07:29:09,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741918_1096 (size=4875) 2024-11-25T07:29:09,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741918_1096 (size=4875) 2024-11-25T07:29:09,355 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732519749355/Put/seqid=0 2024-11-25T07:29:09,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741919_1097 (size=4875) 2024-11-25T07:29:09,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741919_1097 (size=4875) 2024-11-25T07:29:09,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741919_1097 (size=4875) 2024-11-25T07:29:09,371 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732519749371/Put/seqid=0 2024-11-25T07:29:09,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741920_1098 (size=4875) 2024-11-25T07:29:09,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741920_1098 (size=4875) 2024-11-25T07:29:09,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741920_1098 (size=4875) 2024-11-25T07:29:09,378 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44261/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 273fc93dcce824ae865261ee6fdcfafd/a 2024-11-25T07:29:09,382 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-25T07:29:09,382 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-25T07:29:09,382 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44261/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 273fc93dcce824ae865261ee6fdcfafd/a 2024-11-25T07:29:09,386 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-25T07:29:09,386 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-25T07:29:09,386 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44261/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 273fc93dcce824ae865261ee6fdcfafd/a 2024-11-25T07:29:09,390 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-25T07:29:09,390 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-25T07:29:09,390 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 273fc93dcce824ae865261ee6fdcfafd 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-25T07:29:09,413 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/.tmp/a/240a114d8576457db99254fa28a0bbf5 is 55, key is testCompactedBulkLoadedFiles/a:a/1732519749337/Put/seqid=0 2024-11-25T07:29:09,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741921_1099 (size=5107) 2024-11-25T07:29:09,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741921_1099 (size=5107) 2024-11-25T07:29:09,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741921_1099 (size=5107) 2024-11-25T07:29:09,422 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/.tmp/a/240a114d8576457db99254fa28a0bbf5 2024-11-25T07:29:09,429 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/.tmp/a/240a114d8576457db99254fa28a0bbf5 as hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/240a114d8576457db99254fa28a0bbf5 2024-11-25T07:29:09,434 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/240a114d8576457db99254fa28a0bbf5, entries=1, sequenceid=4, filesize=5.0 K 2024-11-25T07:29:09,435 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 273fc93dcce824ae865261ee6fdcfafd in 45ms, sequenceid=4, compaction requested=false 2024-11-25T07:29:09,435 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 273fc93dcce824ae865261ee6fdcfafd: 2024-11-25T07:29:09,437 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/5530142da02640a3922b7c7343e97024_SeqId_4_ 2024-11-25T07:29:09,438 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/881e985b3d204b6eae9f3f77a050a06b_SeqId_4_ 2024-11-25T07:29:09,439 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/e40b3f70605342bbafac9da340270015_SeqId_4_ 2024-11-25T07:29:09,439 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44261/hbase/testCompactedBulkLoadedFiles/hfile0 into 273fc93dcce824ae865261ee6fdcfafd/a as hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/5530142da02640a3922b7c7343e97024_SeqId_4_ - updating store file list. 2024-11-25T07:29:09,443 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5530142da02640a3922b7c7343e97024_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T07:29:09,443 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/5530142da02640a3922b7c7343e97024_SeqId_4_ into 273fc93dcce824ae865261ee6fdcfafd/a 2024-11-25T07:29:09,443 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44261/hbase/testCompactedBulkLoadedFiles/hfile0 into 273fc93dcce824ae865261ee6fdcfafd/a (new location: hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/5530142da02640a3922b7c7343e97024_SeqId_4_) 2024-11-25T07:29:09,444 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44261/hbase/testCompactedBulkLoadedFiles/hfile1 into 273fc93dcce824ae865261ee6fdcfafd/a as hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/881e985b3d204b6eae9f3f77a050a06b_SeqId_4_ - updating store file list. 2024-11-25T07:29:09,449 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 881e985b3d204b6eae9f3f77a050a06b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T07:29:09,449 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/881e985b3d204b6eae9f3f77a050a06b_SeqId_4_ into 273fc93dcce824ae865261ee6fdcfafd/a 2024-11-25T07:29:09,449 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44261/hbase/testCompactedBulkLoadedFiles/hfile1 into 273fc93dcce824ae865261ee6fdcfafd/a (new location: hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/881e985b3d204b6eae9f3f77a050a06b_SeqId_4_) 2024-11-25T07:29:09,450 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44261/hbase/testCompactedBulkLoadedFiles/hfile2 into 273fc93dcce824ae865261ee6fdcfafd/a as hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/e40b3f70605342bbafac9da340270015_SeqId_4_ - updating store file list. 2024-11-25T07:29:09,454 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for e40b3f70605342bbafac9da340270015_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T07:29:09,454 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/e40b3f70605342bbafac9da340270015_SeqId_4_ into 273fc93dcce824ae865261ee6fdcfafd/a 2024-11-25T07:29:09,454 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44261/hbase/testCompactedBulkLoadedFiles/hfile2 into 273fc93dcce824ae865261ee6fdcfafd/a (new location: hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/e40b3f70605342bbafac9da340270015_SeqId_4_) 2024-11-25T07:29:09,462 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T07:29:09,462 DEBUG [Time-limited test {}] regionserver.HStore(1541): 273fc93dcce824ae865261ee6fdcfafd/a is initiating major compaction (all files) 2024-11-25T07:29:09,462 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 273fc93dcce824ae865261ee6fdcfafd/a in testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd. 2024-11-25T07:29:09,462 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/240a114d8576457db99254fa28a0bbf5, hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/5530142da02640a3922b7c7343e97024_SeqId_4_, hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/881e985b3d204b6eae9f3f77a050a06b_SeqId_4_, hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/e40b3f70605342bbafac9da340270015_SeqId_4_] into tmpdir=hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/.tmp, totalSize=19.3 K 2024-11-25T07:29:09,463 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 240a114d8576457db99254fa28a0bbf5, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732519749337 2024-11-25T07:29:09,463 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5530142da02640a3922b7c7343e97024_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-25T07:29:09,463 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 881e985b3d204b6eae9f3f77a050a06b_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-25T07:29:09,464 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e40b3f70605342bbafac9da340270015_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-25T07:29:09,474 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/.tmp/a/73e42a1545df48cebe4bb3bc0af507d8 is 55, key is testCompactedBulkLoadedFiles/a:a/1732519749337/Put/seqid=0 2024-11-25T07:29:09,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741922_1100 (size=6154) 2024-11-25T07:29:09,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741922_1100 (size=6154) 2024-11-25T07:29:09,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741922_1100 (size=6154) 2024-11-25T07:29:09,492 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/.tmp/a/73e42a1545df48cebe4bb3bc0af507d8 as hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/73e42a1545df48cebe4bb3bc0af507d8 2024-11-25T07:29:09,499 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 273fc93dcce824ae865261ee6fdcfafd/a of 273fc93dcce824ae865261ee6fdcfafd into 73e42a1545df48cebe4bb3bc0af507d8(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T07:29:09,499 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 273fc93dcce824ae865261ee6fdcfafd: 2024-11-25T07:29:09,499 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-25T07:29:09,499 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-25T07:29:09,534 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44261/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304, size=0 (0bytes) 2024-11-25T07:29:09,534 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44261/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304 might be still open, length is 0 2024-11-25T07:29:09,534 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44261/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304 2024-11-25T07:29:09,535 WARN [IPC Server handler 0 on default port 44261 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-25T07:29:09,535 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44261/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304 after 1ms 2024-11-25T07:29:10,529 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-25T07:29:10,530 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-25T07:29:10,531 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-25T07:29:10,531 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-25T07:29:12,090 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:58668 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:37227:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58668 dst: /127.0.0.1:37227 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37227 remote=/127.0.0.1:58668]. Total timeout mills is 60000, 57409 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:12,090 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:40874 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:37043:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40874 dst: /127.0.0.1:37043 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:12,090 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:38280 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33603:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38280 dst: /127.0.0.1:33603 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:12,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741917_1101 (size=1659) 2024-11-25T07:29:12,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741917_1101 (size=1659) 2024-11-25T07:29:12,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741917_1101 (size=1659) 2024-11-25T07:29:13,536 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44261/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304 after 4002ms 2024-11-25T07:29:13,539 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44261/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304 took 4005ms 2024-11-25T07:29:13,541 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44261/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304; continuing. 2024-11-25T07:29:13,541 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44261/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304 so closing down 2024-11-25T07:29:13,541 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T07:29:13,542 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732519749304.temp 2024-11-25T07:29:13,545 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/recovered.edits/0000000000000000003-wal.1732519749304.temp 2024-11-25T07:29:13,545 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T07:29:13,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741923_1102 (size=698) 2024-11-25T07:29:13,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741923_1102 (size=698) 2024-11-25T07:29:13,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741923_1102 (size=698) 2024-11-25T07:29:13,554 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/recovered.edits/0000000000000000003-wal.1732519749304.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-25T07:29:13,555 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/recovered.edits/0000000000000000003-wal.1732519749304.temp to hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/recovered.edits/0000000000000000008 2024-11-25T07:29:13,555 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:44261/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304, size=0, length=0, corrupted=false, cancelled=false 2024-11-25T07:29:13,555 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44261/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304, journal: Splitting hdfs://localhost:44261/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304, size=0 (0bytes) at 1732519749534Finishing writing output for hdfs://localhost:44261/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304 so closing down at 1732519753541 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/recovered.edits/0000000000000000003-wal.1732519749304.temp at 1732519753545 (+4 ms)3 split writer threads finished at 1732519753545Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/recovered.edits/0000000000000000003-wal.1732519749304.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732519753554 (+9 ms)Rename recovered edits hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/recovered.edits/0000000000000000003-wal.1732519749304.temp to hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/recovered.edits/0000000000000000008 at 1732519753555 (+1 ms)Processed 5 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:44261/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304, size=0, length=0, corrupted=false, cancelled=false at 1732519753555 2024-11-25T07:29:13,557 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44261/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304 to hdfs://localhost:44261/hbase/oldWALs/wal.1732519749304 2024-11-25T07:29:13,558 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/recovered.edits/0000000000000000008 2024-11-25T07:29:13,558 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:29:13,560 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:29:13,574 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519753560, exclude list is [], retry=0 2024-11-25T07:29:13,577 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:29:13,577 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:29:13,577 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:29:13,579 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519753560 2024-11-25T07:29:13,579 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-25T07:29:13,579 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 273fc93dcce824ae865261ee6fdcfafd, NAME => 'testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:29:13,579 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:29:13,579 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:13,580 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:13,581 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:13,582 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 273fc93dcce824ae865261ee6fdcfafd columnFamilyName a 2024-11-25T07:29:13,582 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:13,592 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/240a114d8576457db99254fa28a0bbf5 2024-11-25T07:29:13,596 DEBUG [StoreFileOpener-273fc93dcce824ae865261ee6fdcfafd-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5530142da02640a3922b7c7343e97024_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T07:29:13,596 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/5530142da02640a3922b7c7343e97024_SeqId_4_ 2024-11-25T07:29:13,599 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/73e42a1545df48cebe4bb3bc0af507d8 2024-11-25T07:29:13,603 DEBUG [StoreFileOpener-273fc93dcce824ae865261ee6fdcfafd-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 881e985b3d204b6eae9f3f77a050a06b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T07:29:13,604 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/881e985b3d204b6eae9f3f77a050a06b_SeqId_4_ 2024-11-25T07:29:13,607 DEBUG [StoreFileOpener-273fc93dcce824ae865261ee6fdcfafd-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for e40b3f70605342bbafac9da340270015_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T07:29:13,607 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/e40b3f70605342bbafac9da340270015_SeqId_4_ 2024-11-25T07:29:13,608 WARN [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/240a114d8576457db99254fa28a0bbf5 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7284c39c 2024-11-25T07:29:13,608 WARN [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/5530142da02640a3922b7c7343e97024_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7284c39c 2024-11-25T07:29:13,608 WARN [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/881e985b3d204b6eae9f3f77a050a06b_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7284c39c 2024-11-25T07:29:13,608 WARN [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/e40b3f70605342bbafac9da340270015_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7284c39c 2024-11-25T07:29:13,608 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/240a114d8576457db99254fa28a0bbf5, hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/5530142da02640a3922b7c7343e97024_SeqId_4_, hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/881e985b3d204b6eae9f3f77a050a06b_SeqId_4_, hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/e40b3f70605342bbafac9da340270015_SeqId_4_] to archive 2024-11-25T07:29:13,609 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T07:29:13,611 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/240a114d8576457db99254fa28a0bbf5 to hdfs://localhost:44261/hbase/archive/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/240a114d8576457db99254fa28a0bbf5 2024-11-25T07:29:13,612 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/5530142da02640a3922b7c7343e97024_SeqId_4_ to hdfs://localhost:44261/hbase/archive/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/5530142da02640a3922b7c7343e97024_SeqId_4_ 2024-11-25T07:29:13,613 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/881e985b3d204b6eae9f3f77a050a06b_SeqId_4_ to hdfs://localhost:44261/hbase/archive/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/881e985b3d204b6eae9f3f77a050a06b_SeqId_4_ 2024-11-25T07:29:13,614 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/e40b3f70605342bbafac9da340270015_SeqId_4_ to hdfs://localhost:44261/hbase/archive/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/a/e40b3f70605342bbafac9da340270015_SeqId_4_ 2024-11-25T07:29:13,614 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(327): Store=273fc93dcce824ae865261ee6fdcfafd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:13,615 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:13,615 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 273fc93dcce824ae865261ee6fdcfafd columnFamilyName b 2024-11-25T07:29:13,615 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:13,616 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(327): Store=273fc93dcce824ae865261ee6fdcfafd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:13,616 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:13,617 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 273fc93dcce824ae865261ee6fdcfafd columnFamilyName c 2024-11-25T07:29:13,617 DEBUG [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:13,617 INFO [StoreOpener-273fc93dcce824ae865261ee6fdcfafd-1 {}] regionserver.HStore(327): Store=273fc93dcce824ae865261ee6fdcfafd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:13,617 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:13,618 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:13,620 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:13,620 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/recovered.edits/0000000000000000008 2024-11-25T07:29:13,624 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 273fc93dcce824ae865261ee6fdcfafd : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "273fc93dcce824ae865261ee6fdcfafd" family_name: "a" compaction_input: "240a114d8576457db99254fa28a0bbf5" compaction_input: "5530142da02640a3922b7c7343e97024_SeqId_4_" compaction_input: "881e985b3d204b6eae9f3f77a050a06b_SeqId_4_" compaction_input: "e40b3f70605342bbafac9da340270015_SeqId_4_" compaction_output: "73e42a1545df48cebe4bb3bc0af507d8" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-25T07:29:13,625 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-25T07:29:13,625 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/recovered.edits/0000000000000000008 2024-11-25T07:29:13,626 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/recovered.edits/0000000000000000008 2024-11-25T07:29:13,627 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:13,627 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:13,628 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:29:13,629 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 273fc93dcce824ae865261ee6fdcfafd 2024-11-25T07:29:13,631 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testCompactedBulkLoadedFiles/273fc93dcce824ae865261ee6fdcfafd/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-25T07:29:13,633 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 273fc93dcce824ae865261ee6fdcfafd; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68054508, jitterRate=0.014091193675994873}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:29:13,633 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 273fc93dcce824ae865261ee6fdcfafd: Writing region info on filesystem at 1732519753580Initializing all the Stores at 1732519753580Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519753580Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519753581 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519753581Cleaning up temporary data from old regions at 1732519753627 (+46 ms)Region opened successfully at 1732519753633 (+6 ms) 2024-11-25T07:29:13,635 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 273fc93dcce824ae865261ee6fdcfafd, disabling compactions & flushes 2024-11-25T07:29:13,635 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd. 2024-11-25T07:29:13,635 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd. 2024-11-25T07:29:13,635 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd. after waiting 0 ms 2024-11-25T07:29:13,635 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd. 2024-11-25T07:29:13,636 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732519749233.273fc93dcce824ae865261ee6fdcfafd. 2024-11-25T07:29:13,636 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 273fc93dcce824ae865261ee6fdcfafd: Waiting for close lock at 1732519753635Disabling compacts and flushes for region at 1732519753635Disabling writes for close at 1732519753635Writing region close event to WAL at 1732519753636 (+1 ms)Closed at 1732519753636 2024-11-25T07:29:13,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741924_1103 (size=93) 2024-11-25T07:29:13,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741924_1103 (size=93) 2024-11-25T07:29:13,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741924_1103 (size=93) 2024-11-25T07:29:13,642 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T07:29:13,642 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732519753560) 2024-11-25T07:29:13,658 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=437 (was 423) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1617308189_22 at /127.0.0.1:50474 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1617308189_22 at /127.0.0.1:49958 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (473829079) connection to localhost/127.0.0.1:44261 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:44261 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1617308189_22 at /127.0.0.1:58096 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1342 (was 1261) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 117) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8197 (was 8218) 2024-11-25T07:29:13,658 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1342 is superior to 1024 2024-11-25T07:29:13,672 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=437, OpenFileDescriptor=1342, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=8195 2024-11-25T07:29:13,672 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1342 is superior to 1024 2024-11-25T07:29:13,689 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:29:13,691 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:29:13,691 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T07:29:13,693 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-19981125, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/hregion-19981125, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:29:13,708 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-19981125/hregion-19981125.1732519753694, exclude list is [], retry=0 2024-11-25T07:29:13,711 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:29:13,711 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:29:13,711 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:29:13,713 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-19981125/hregion-19981125.1732519753694 2024-11-25T07:29:13,714 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:37263:37263)] 2024-11-25T07:29:13,714 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 24cf4ed86d0d61ac9ffcf3f3106107ec, NAME => 'testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44261/hbase 2024-11-25T07:29:13,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741926_1105 (size=67) 2024-11-25T07:29:13,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741926_1105 (size=67) 2024-11-25T07:29:13,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741926_1105 (size=67) 2024-11-25T07:29:13,725 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:29:13,726 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,728 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24cf4ed86d0d61ac9ffcf3f3106107ec columnFamilyName a 2024-11-25T07:29:13,728 DEBUG [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:13,729 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(327): Store=24cf4ed86d0d61ac9ffcf3f3106107ec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:13,729 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,730 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24cf4ed86d0d61ac9ffcf3f3106107ec columnFamilyName b 2024-11-25T07:29:13,731 DEBUG [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:13,731 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(327): Store=24cf4ed86d0d61ac9ffcf3f3106107ec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:13,731 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,732 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24cf4ed86d0d61ac9ffcf3f3106107ec columnFamilyName c 2024-11-25T07:29:13,733 DEBUG [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:13,733 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(327): Store=24cf4ed86d0d61ac9ffcf3f3106107ec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:13,733 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,734 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,734 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,735 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,735 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,736 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:29:13,737 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,739 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:29:13,740 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 24cf4ed86d0d61ac9ffcf3f3106107ec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74953867, jitterRate=0.11689965426921844}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:29:13,740 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 24cf4ed86d0d61ac9ffcf3f3106107ec: Writing region info on filesystem at 1732519753725Initializing all the Stores at 1732519753726 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519753726Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519753726Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519753726Cleaning up temporary data from old regions at 1732519753735 (+9 ms)Region opened successfully at 1732519753740 (+5 ms) 2024-11-25T07:29:13,740 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 24cf4ed86d0d61ac9ffcf3f3106107ec, disabling compactions & flushes 2024-11-25T07:29:13,740 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. 2024-11-25T07:29:13,740 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. 2024-11-25T07:29:13,740 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. after waiting 0 ms 2024-11-25T07:29:13,740 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. 2024-11-25T07:29:13,741 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. 2024-11-25T07:29:13,741 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 24cf4ed86d0d61ac9ffcf3f3106107ec: Waiting for close lock at 1732519753740Disabling compacts and flushes for region at 1732519753740Disabling writes for close at 1732519753740Writing region close event to WAL at 1732519753741 (+1 ms)Closed at 1732519753741 2024-11-25T07:29:13,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741925_1104 (size=93) 2024-11-25T07:29:13,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741925_1104 (size=93) 2024-11-25T07:29:13,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741925_1104 (size=93) 2024-11-25T07:29:13,746 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T07:29:13,746 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-19981125:(num 1732519753694) 2024-11-25T07:29:13,747 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:29:13,748 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:29:13,763 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753749, exclude list is [], retry=0 2024-11-25T07:29:13,766 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:29:13,766 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:29:13,767 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:29:13,768 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753749 2024-11-25T07:29:13,769 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:37263:37263)] 2024-11-25T07:29:13,769 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 24cf4ed86d0d61ac9ffcf3f3106107ec, NAME => 'testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:29:13,769 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:29:13,770 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,770 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,771 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,772 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24cf4ed86d0d61ac9ffcf3f3106107ec columnFamilyName a 2024-11-25T07:29:13,772 DEBUG [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:13,772 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(327): Store=24cf4ed86d0d61ac9ffcf3f3106107ec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:13,772 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,773 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24cf4ed86d0d61ac9ffcf3f3106107ec columnFamilyName b 2024-11-25T07:29:13,773 DEBUG [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:13,773 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(327): Store=24cf4ed86d0d61ac9ffcf3f3106107ec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:13,774 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,774 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24cf4ed86d0d61ac9ffcf3f3106107ec columnFamilyName c 2024-11-25T07:29:13,774 DEBUG [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:13,775 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(327): Store=24cf4ed86d0d61ac9ffcf3f3106107ec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:13,775 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,775 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,776 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,777 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,777 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,778 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:29:13,779 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,780 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 24cf4ed86d0d61ac9ffcf3f3106107ec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70751133, jitterRate=0.05427403748035431}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:29:13,780 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 24cf4ed86d0d61ac9ffcf3f3106107ec: Writing region info on filesystem at 1732519753770Initializing all the Stores at 1732519753770Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519753771 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519753771Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519753771Cleaning up temporary data from old regions at 1732519753777 (+6 ms)Region opened successfully at 1732519753780 (+3 ms) 2024-11-25T07:29:13,794 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-25T07:29:13,812 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/a/d9efec17e02f4e3793872c67c4c06ec5 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732519753780/Put/seqid=0 2024-11-25T07:29:13,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741928_1107 (size=5958) 2024-11-25T07:29:13,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741928_1107 (size=5958) 2024-11-25T07:29:13,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741928_1107 (size=5958) 2024-11-25T07:29:13,820 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/a/d9efec17e02f4e3793872c67c4c06ec5 2024-11-25T07:29:13,826 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/a/d9efec17e02f4e3793872c67c4c06ec5 as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/a/d9efec17e02f4e3793872c67c4c06ec5 2024-11-25T07:29:13,831 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/a/d9efec17e02f4e3793872c67c4c06ec5, entries=10, sequenceid=13, filesize=5.8 K 2024-11-25T07:29:13,832 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 24cf4ed86d0d61ac9ffcf3f3106107ec in 38ms, sequenceid=13, compaction requested=false 2024-11-25T07:29:13,832 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 24cf4ed86d0d61ac9ffcf3f3106107ec: 2024-11-25T07:29:13,837 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T07:29:13,888 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 24cf4ed86d0d61ac9ffcf3f3106107ec, disabling compactions & flushes 2024-11-25T07:29:13,888 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. 2024-11-25T07:29:13,888 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. 2024-11-25T07:29:13,888 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. after waiting 0 ms 2024-11-25T07:29:13,889 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. 2024-11-25T07:29:13,893 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. 2024-11-25T07:29:13,893 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. 2024-11-25T07:29:13,893 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 24cf4ed86d0d61ac9ffcf3f3106107ec: Waiting for close lock at 1732519753888Disabling compacts and flushes for region at 1732519753888Disabling writes for close at 1732519753888Writing region close event to WAL at 1732519753892 (+4 ms)Closed at 1732519753893 (+1 ms) 2024-11-25T07:29:13,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741927_1106 (size=5869) 2024-11-25T07:29:13,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741927_1106 (size=5869) 2024-11-25T07:29:13,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741927_1106 (size=5869) 2024-11-25T07:29:13,921 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753749, size=5.7 K (5869bytes) 2024-11-25T07:29:13,921 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753749 2024-11-25T07:29:13,922 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753749 after 0ms 2024-11-25T07:29:13,925 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753749 took 4ms 2024-11-25T07:29:13,926 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753749 so closing down 2024-11-25T07:29:13,927 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T07:29:13,928 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732519753749.temp 2024-11-25T07:29:13,929 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000003-wal.1732519753749.temp 2024-11-25T07:29:13,929 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T07:29:13,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741929_1108 (size=5223) 2024-11-25T07:29:13,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741929_1108 (size=5223) 2024-11-25T07:29:13,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741929_1108 (size=5223) 2024-11-25T07:29:13,938 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000003-wal.1732519753749.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-25T07:29:13,939 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000003-wal.1732519753749.temp to hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000035 2024-11-25T07:29:13,939 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753749, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-11-25T07:29:13,939 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753749, journal: Splitting hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753749, size=5.7 K (5869bytes) at 1732519753921Finishing writing output for hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753749 so closing down at 1732519753926 (+5 ms)Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000003-wal.1732519753749.temp at 1732519753929 (+3 ms)3 split writer threads finished at 1732519753929Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000003-wal.1732519753749.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732519753938 (+9 ms)Rename recovered edits hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000003-wal.1732519753749.temp to hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000035 at 1732519753939 (+1 ms)Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753749, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1732519753939 2024-11-25T07:29:13,941 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753749 to hdfs://localhost:44261/hbase/oldWALs/wal.1732519753749 2024-11-25T07:29:13,942 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000035 2024-11-25T07:29:13,942 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:29:13,944 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:29:13,965 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945, exclude list is [], retry=0 2024-11-25T07:29:13,969 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:29:13,970 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:29:13,970 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:29:13,974 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945 2024-11-25T07:29:13,974 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:43429:43429)] 2024-11-25T07:29:13,974 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 24cf4ed86d0d61ac9ffcf3f3106107ec, NAME => 'testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:29:13,975 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:29:13,975 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,975 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,976 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,977 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24cf4ed86d0d61ac9ffcf3f3106107ec columnFamilyName a 2024-11-25T07:29:13,977 DEBUG [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:13,984 DEBUG [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/a/d9efec17e02f4e3793872c67c4c06ec5 2024-11-25T07:29:13,984 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(327): Store=24cf4ed86d0d61ac9ffcf3f3106107ec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:13,984 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,985 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24cf4ed86d0d61ac9ffcf3f3106107ec columnFamilyName b 2024-11-25T07:29:13,985 DEBUG [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:13,986 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(327): Store=24cf4ed86d0d61ac9ffcf3f3106107ec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:13,986 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,986 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24cf4ed86d0d61ac9ffcf3f3106107ec columnFamilyName c 2024-11-25T07:29:13,987 DEBUG [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:13,987 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(327): Store=24cf4ed86d0d61ac9ffcf3f3106107ec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:13,987 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,988 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,989 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:13,990 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000035 2024-11-25T07:29:13,993 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000035 2024-11-25T07:29:13,993 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-25T07:29:14,015 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/b/86635e4c2acf422dba47372132262367 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732519753832/Put/seqid=0 2024-11-25T07:29:14,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741931_1110 (size=5958) 2024-11-25T07:29:14,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741931_1110 (size=5958) 2024-11-25T07:29:14,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741931_1110 (size=5958) 2024-11-25T07:29:14,027 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/b/86635e4c2acf422dba47372132262367 2024-11-25T07:29:14,047 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/c/8ab6f56a63cc419289cc713508256e47 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732519753840/Put/seqid=0 2024-11-25T07:29:14,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741932_1111 (size=5958) 2024-11-25T07:29:14,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741932_1111 (size=5958) 2024-11-25T07:29:14,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741932_1111 (size=5958) 2024-11-25T07:29:14,055 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/c/8ab6f56a63cc419289cc713508256e47 2024-11-25T07:29:14,060 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/b/86635e4c2acf422dba47372132262367 as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/b/86635e4c2acf422dba47372132262367 2024-11-25T07:29:14,065 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/b/86635e4c2acf422dba47372132262367, entries=10, sequenceid=35, filesize=5.8 K 2024-11-25T07:29:14,065 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/c/8ab6f56a63cc419289cc713508256e47 as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/c/8ab6f56a63cc419289cc713508256e47 2024-11-25T07:29:14,070 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/c/8ab6f56a63cc419289cc713508256e47, entries=10, sequenceid=35, filesize=5.8 K 2024-11-25T07:29:14,070 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 24cf4ed86d0d61ac9ffcf3f3106107ec in 77ms, sequenceid=35, compaction requested=false; wal=null 2024-11-25T07:29:14,071 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000035 2024-11-25T07:29:14,072 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:14,072 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:14,072 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:29:14,074 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:14,076 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-25T07:29:14,077 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 24cf4ed86d0d61ac9ffcf3f3106107ec; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70819801, jitterRate=0.05529727041721344}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:29:14,077 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 24cf4ed86d0d61ac9ffcf3f3106107ec: Writing region info on filesystem at 1732519753975Initializing all the Stores at 1732519753976 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519753976Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519753976Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519753976Obtaining lock to block concurrent updates at 1732519753993 (+17 ms)Preparing flush snapshotting stores in 24cf4ed86d0d61ac9ffcf3f3106107ec at 1732519753993Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732519753993Flushing stores of testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. at 1732519753993Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec/b: creating writer at 1732519753993Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec/b: appending metadata at 1732519754014 (+21 ms)Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec/b: closing flushed file at 1732519754014Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec/c: creating writer at 1732519754032 (+18 ms)Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec/c: appending metadata at 1732519754046 (+14 ms)Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec/c: closing flushed file at 1732519754046Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31926081: reopening flushed file at 1732519754059 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b2035f0: reopening flushed file at 1732519754065 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 24cf4ed86d0d61ac9ffcf3f3106107ec in 77ms, sequenceid=35, compaction requested=false; wal=null at 1732519754070 (+5 ms)Cleaning up temporary data from old regions at 1732519754072 (+2 ms)Region opened successfully at 1732519754077 (+5 ms) 2024-11-25T07:29:14,158 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945, size=0 (0bytes) 2024-11-25T07:29:14,158 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945 might be still open, length is 0 2024-11-25T07:29:14,158 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945 2024-11-25T07:29:14,158 WARN [IPC Server handler 3 on default port 44261 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-25T07:29:14,158 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945 after 0ms 2024-11-25T07:29:15,088 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:50530 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33603:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50530 dst: /127.0.0.1:33603 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33603 remote=/127.0.0.1:50530]. Total timeout mills is 60000, 59019 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:15,089 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:58148 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:37227:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58148 dst: /127.0.0.1:37227 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:15,089 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1862095393_22 at /127.0.0.1:50016 [Receiving block BP-661403191-172.17.0.2-1732519717195:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:37043:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50016 dst: /127.0.0.1:37043 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:15,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741930_1112 (size=5215) 2024-11-25T07:29:15,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741930_1112 (size=5215) 2024-11-25T07:29:15,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741930_1112 (size=5215) 2024-11-25T07:29:18,159 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945 after 4001ms 2024-11-25T07:29:18,162 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945 took 4005ms 2024-11-25T07:29:18,164 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945; continuing. 2024-11-25T07:29:18,164 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945 so closing down 2024-11-25T07:29:18,164 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T07:29:18,165 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732519753945.temp 2024-11-25T07:29:18,166 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000037-wal.1732519753945.temp 2024-11-25T07:29:18,166 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T07:29:18,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741933_1113 (size=5223) 2024-11-25T07:29:18,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741933_1113 (size=5223) 2024-11-25T07:29:18,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741933_1113 (size=5223) 2024-11-25T07:29:18,174 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000037-wal.1732519753945.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-25T07:29:18,175 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000037-wal.1732519753945.temp to hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000066 2024-11-25T07:29:18,175 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945, size=0, length=0, corrupted=false, cancelled=false 2024-11-25T07:29:18,175 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945, journal: Splitting hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945, size=0 (0bytes) at 1732519754158Finishing writing output for hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945 so closing down at 1732519758164 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000037-wal.1732519753945.temp at 1732519758166 (+2 ms)3 split writer threads finished at 1732519758166Closed recovered edits writer path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000037-wal.1732519753945.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732519758174 (+8 ms)Rename recovered edits hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000037-wal.1732519753945.temp to hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000066 at 1732519758175 (+1 ms)Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945, size=0, length=0, corrupted=false, cancelled=false at 1732519758175 2024-11-25T07:29:18,176 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945 to hdfs://localhost:44261/hbase/oldWALs/wal.1732519753945 2024-11-25T07:29:18,177 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000066 2024-11-25T07:29:18,177 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T07:29:18,179 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44261/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688, archiveDir=hdfs://localhost:44261/hbase/oldWALs, maxLogs=32 2024-11-25T07:29:18,192 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519758179, exclude list is [], retry=0 2024-11-25T07:29:18,194 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37227,DS-ff8ed39c-98df-48c1-a178-6da2e533ccdc,DISK] 2024-11-25T07:29:18,195 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33603,DS-e6366611-6e9d-43b1-97d0-a9139f2a6b18,DISK] 2024-11-25T07:29:18,195 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37043,DS-533c0c67-8243-4b60-b744-302e9d9fc36e,DISK] 2024-11-25T07:29:18,196 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519758179 2024-11-25T07:29:18,196 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43429:43429),(127.0.0.1/127.0.0.1:37263:37263),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-25T07:29:18,197 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:29:18,198 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:18,199 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24cf4ed86d0d61ac9ffcf3f3106107ec columnFamilyName a 2024-11-25T07:29:18,199 DEBUG [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:18,203 DEBUG [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/a/d9efec17e02f4e3793872c67c4c06ec5 2024-11-25T07:29:18,203 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(327): Store=24cf4ed86d0d61ac9ffcf3f3106107ec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:18,203 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:18,204 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24cf4ed86d0d61ac9ffcf3f3106107ec columnFamilyName b 2024-11-25T07:29:18,204 DEBUG [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:18,208 DEBUG [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/b/86635e4c2acf422dba47372132262367 2024-11-25T07:29:18,208 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(327): Store=24cf4ed86d0d61ac9ffcf3f3106107ec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:18,208 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:18,208 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24cf4ed86d0d61ac9ffcf3f3106107ec columnFamilyName c 2024-11-25T07:29:18,209 DEBUG [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:29:18,213 DEBUG [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/c/8ab6f56a63cc419289cc713508256e47 2024-11-25T07:29:18,213 INFO [StoreOpener-24cf4ed86d0d61ac9ffcf3f3106107ec-1 {}] regionserver.HStore(327): Store=24cf4ed86d0d61ac9ffcf3f3106107ec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:29:18,213 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:18,214 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:18,215 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:18,215 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000066 2024-11-25T07:29:18,222 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000066 2024-11-25T07:29:18,222 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-25T07:29:18,237 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/a/d72c010463e748d0bcfe3864e370b173 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732519754085/Put/seqid=0 2024-11-25T07:29:18,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741935_1115 (size=5958) 2024-11-25T07:29:18,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741935_1115 (size=5958) 2024-11-25T07:29:18,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741935_1115 (size=5958) 2024-11-25T07:29:18,243 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/a/d72c010463e748d0bcfe3864e370b173 2024-11-25T07:29:18,263 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/b/0559d4229d294f00925052d557fb5212 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732519754094/Put/seqid=0 2024-11-25T07:29:18,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741936_1116 (size=5958) 2024-11-25T07:29:18,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741936_1116 (size=5958) 2024-11-25T07:29:18,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741936_1116 (size=5958) 2024-11-25T07:29:18,270 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/b/0559d4229d294f00925052d557fb5212 2024-11-25T07:29:18,289 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/c/5115bacc75a24656ad7b30ea80979cda is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732519754101/Put/seqid=0 2024-11-25T07:29:18,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741937_1117 (size=5958) 2024-11-25T07:29:18,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741937_1117 (size=5958) 2024-11-25T07:29:18,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741937_1117 (size=5958) 2024-11-25T07:29:18,295 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/c/5115bacc75a24656ad7b30ea80979cda 2024-11-25T07:29:18,300 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/a/d72c010463e748d0bcfe3864e370b173 as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/a/d72c010463e748d0bcfe3864e370b173 2024-11-25T07:29:18,304 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/a/d72c010463e748d0bcfe3864e370b173, entries=10, sequenceid=66, filesize=5.8 K 2024-11-25T07:29:18,305 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/b/0559d4229d294f00925052d557fb5212 as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/b/0559d4229d294f00925052d557fb5212 2024-11-25T07:29:18,310 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/b/0559d4229d294f00925052d557fb5212, entries=10, sequenceid=66, filesize=5.8 K 2024-11-25T07:29:18,311 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/.tmp/c/5115bacc75a24656ad7b30ea80979cda as hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/c/5115bacc75a24656ad7b30ea80979cda 2024-11-25T07:29:18,316 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/c/5115bacc75a24656ad7b30ea80979cda, entries=10, sequenceid=66, filesize=5.8 K 2024-11-25T07:29:18,316 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 24cf4ed86d0d61ac9ffcf3f3106107ec in 94ms, sequenceid=66, compaction requested=false; wal=null 2024-11-25T07:29:18,317 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/0000000000000000066 2024-11-25T07:29:18,318 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:18,318 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:18,318 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T07:29:18,319 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 24cf4ed86d0d61ac9ffcf3f3106107ec 2024-11-25T07:29:18,321 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/hbase/data/default/testReplayEditsWrittenViaHRegion/24cf4ed86d0d61ac9ffcf3f3106107ec/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-25T07:29:18,322 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 24cf4ed86d0d61ac9ffcf3f3106107ec; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74255721, jitterRate=0.10649646818637848}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T07:29:18,322 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 24cf4ed86d0d61ac9ffcf3f3106107ec: Writing region info on filesystem at 1732519758197Initializing all the Stores at 1732519758197Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519758198 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519758198Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732519758198Obtaining lock to block concurrent updates at 1732519758222 (+24 ms)Preparing flush snapshotting stores in 24cf4ed86d0d61ac9ffcf3f3106107ec at 1732519758222Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732519758222Flushing stores of testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. at 1732519758222Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec/a: creating writer at 1732519758222Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec/a: appending metadata at 1732519758236 (+14 ms)Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec/a: closing flushed file at 1732519758236Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec/b: creating writer at 1732519758248 (+12 ms)Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec/b: appending metadata at 1732519758263 (+15 ms)Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec/b: closing flushed file at 1732519758263Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec/c: creating writer at 1732519758274 (+11 ms)Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec/c: appending metadata at 1732519758288 (+14 ms)Flushing 24cf4ed86d0d61ac9ffcf3f3106107ec/c: closing flushed file at 1732519758288Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bbbf542: reopening flushed file at 1732519758299 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7216f139: reopening flushed file at 1732519758304 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25312e10: reopening flushed file at 1732519758311 (+7 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 24cf4ed86d0d61ac9ffcf3f3106107ec in 94ms, sequenceid=66, compaction requested=false; wal=null at 1732519758316 (+5 ms)Cleaning up temporary data from old regions at 1732519758318 (+2 ms)Region opened successfully at 1732519758322 (+4 ms) 2024-11-25T07:29:18,335 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 24cf4ed86d0d61ac9ffcf3f3106107ec, disabling compactions & flushes 2024-11-25T07:29:18,336 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. 2024-11-25T07:29:18,336 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. 2024-11-25T07:29:18,336 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. after waiting 0 ms 2024-11-25T07:29:18,336 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. 2024-11-25T07:29:18,337 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732519753689.24cf4ed86d0d61ac9ffcf3f3106107ec. 2024-11-25T07:29:18,337 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 24cf4ed86d0d61ac9ffcf3f3106107ec: Waiting for close lock at 1732519758335Disabling compacts and flushes for region at 1732519758335Disabling writes for close at 1732519758336 (+1 ms)Writing region close event to WAL at 1732519758337 (+1 ms)Closed at 1732519758337 2024-11-25T07:29:18,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741934_1114 (size=93) 2024-11-25T07:29:18,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741934_1114 (size=93) 2024-11-25T07:29:18,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741934_1114 (size=93) 2024-11-25T07:29:18,341 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T07:29:18,341 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732519758179) 2024-11-25T07:29:18,356 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=442 (was 437) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:44261 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_986274244_22 at /127.0.0.1:50556 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (473829079) connection to localhost/127.0.0.1:44261 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_986274244_22 at /127.0.0.1:50062 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_986274244_22 at /127.0.0.1:58200 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1408 (was 1342) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=8109 (was 8195) 2024-11-25T07:29:18,356 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1408 is superior to 1024 2024-11-25T07:29:18,356 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T07:29:18,356 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T07:29:18,356 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T07:29:18,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:29:18,357 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:29:18,357 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T07:29:18,357 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T07:29:18,357 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=208037756, stopped=false 2024-11-25T07:29:18,358 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=5eb3d201e8c9,33903,1732519720118 2024-11-25T07:29:18,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T07:29:18,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T07:29:18,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T07:29:18,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:29:18,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:29:18,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:29:18,360 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T07:29:18,361 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T07:29:18,361 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T07:29:18,361 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T07:29:18,361 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T07:29:18,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:29:18,361 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5eb3d201e8c9,33753,1732519720813' ***** 2024-11-25T07:29:18,362 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T07:29:18,362 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T07:29:18,362 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5eb3d201e8c9,44311,1732519720918' ***** 2024-11-25T07:29:18,362 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T07:29:18,362 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T07:29:18,362 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T07:29:18,362 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T07:29:18,362 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T07:29:18,363 INFO [RS:0;5eb3d201e8c9:33753 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T07:29:18,363 INFO [RS:1;5eb3d201e8c9:44311 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T07:29:18,363 INFO [RS:1;5eb3d201e8c9:44311 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T07:29:18,363 INFO [RS:0;5eb3d201e8c9:33753 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T07:29:18,363 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.HRegionServer(959): stopping server 5eb3d201e8c9,44311,1732519720918 2024-11-25T07:29:18,363 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(3091): Received CLOSE for 0197c07d98552bf4cd477bc9d94b6b62 2024-11-25T07:29:18,363 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T07:29:18,363 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(959): stopping server 5eb3d201e8c9,33753,1732519720813 2024-11-25T07:29:18,363 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T07:29:18,363 INFO [RS:1;5eb3d201e8c9:44311 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;5eb3d201e8c9:44311. 2024-11-25T07:29:18,363 INFO [RS:0;5eb3d201e8c9:33753 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;5eb3d201e8c9:33753. 2024-11-25T07:29:18,363 DEBUG [RS:1;5eb3d201e8c9:44311 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T07:29:18,363 DEBUG [RS:1;5eb3d201e8c9:44311 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:29:18,363 DEBUG [RS:0;5eb3d201e8c9:33753 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T07:29:18,363 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0197c07d98552bf4cd477bc9d94b6b62, disabling compactions & flushes 2024-11-25T07:29:18,363 DEBUG [RS:0;5eb3d201e8c9:33753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:29:18,363 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.HRegionServer(976): stopping server 5eb3d201e8c9,44311,1732519720918; all regions closed. 2024-11-25T07:29:18,363 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:29:18,363 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T07:29:18,363 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:29:18,363 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T07:29:18,363 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. after waiting 0 ms 2024-11-25T07:29:18,363 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T07:29:18,363 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:29:18,363 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T07:29:18,364 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-25T07:29:18,364 DEBUG [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(1325): Online Regions={0197c07d98552bf4cd477bc9d94b6b62=testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62., 1588230740=hbase:meta,,1.1588230740} 2024-11-25T07:29:18,364 DEBUG [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(1351): Waiting on 0197c07d98552bf4cd477bc9d94b6b62, 1588230740 2024-11-25T07:29:18,364 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T07:29:18,365 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T07:29:18,365 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T07:29:18,365 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T07:29:18,365 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T07:29:18,365 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-25T07:29:18,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741835_1011 (size=93) 2024-11-25T07:29:18,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741835_1011 (size=93) 2024-11-25T07:29:18,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741835_1011 (size=93) 2024-11-25T07:29:18,370 DEBUG [RS:1;5eb3d201e8c9:44311 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/oldWALs 2024-11-25T07:29:18,370 INFO [RS:1;5eb3d201e8c9:44311 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5eb3d201e8c9%2C44311%2C1732519720918:(num 1732519722293) 2024-11-25T07:29:18,370 DEBUG [RS:1;5eb3d201e8c9:44311 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:29:18,370 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T07:29:18,370 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/default/testReplayEditsAfterRegionMovedWithMultiCF/0197c07d98552bf4cd477bc9d94b6b62/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-25T07:29:18,371 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T07:29:18,371 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.ChoreService(370): Chore service for: regionserver/5eb3d201e8c9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T07:29:18,371 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T07:29:18,371 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T07:29:18,371 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T07:29:18,371 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T07:29:18,371 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:29:18,371 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0197c07d98552bf4cd477bc9d94b6b62: Waiting for close lock at 1732519758363Running coprocessor pre-close hooks at 1732519758363Disabling compacts and flushes for region at 1732519758363Disabling writes for close at 1732519758363Writing region close event to WAL at 1732519758365 (+2 ms)Running coprocessor post-close hooks at 1732519758371 (+6 ms)Closed at 1732519758371 2024-11-25T07:29:18,371 INFO [RS:1;5eb3d201e8c9:44311 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44311 2024-11-25T07:29:18,371 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62. 2024-11-25T07:29:18,372 INFO [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T07:29:18,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T07:29:18,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5eb3d201e8c9,44311,1732519720918 2024-11-25T07:29:18,373 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T07:29:18,375 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5eb3d201e8c9,44311,1732519720918] 2024-11-25T07:29:18,377 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5eb3d201e8c9,44311,1732519720918 already deleted, retry=false 2024-11-25T07:29:18,377 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5eb3d201e8c9,44311,1732519720918 expired; onlineServers=1 2024-11-25T07:29:18,380 INFO [regionserver/5eb3d201e8c9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T07:29:18,392 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740/.tmp/info/21c0aa379bd04031807a9c2e276f5028 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732519735264.0197c07d98552bf4cd477bc9d94b6b62./info:regioninfo/1732519738257/Put/seqid=0 2024-11-25T07:29:18,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741938_1118 (size=8243) 2024-11-25T07:29:18,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741938_1118 (size=8243) 2024-11-25T07:29:18,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741938_1118 (size=8243) 2024-11-25T07:29:18,447 INFO [regionserver/5eb3d201e8c9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T07:29:18,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T07:29:18,475 INFO [RS:1;5eb3d201e8c9:44311 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T07:29:18,475 INFO [RS:1;5eb3d201e8c9:44311 {}] regionserver.HRegionServer(1031): Exiting; stopping=5eb3d201e8c9,44311,1732519720918; zookeeper connection closed. 2024-11-25T07:29:18,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44311-0x1014e06dc560002, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T07:29:18,475 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2fe369e8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2fe369e8 2024-11-25T07:29:18,565 DEBUG [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T07:29:18,765 DEBUG [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T07:29:18,802 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740/.tmp/info/21c0aa379bd04031807a9c2e276f5028 2024-11-25T07:29:18,823 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740/.tmp/ns/f34515f6c733485681bc7f747ca0223c is 43, key is default/ns:d/1732519722859/Put/seqid=0 2024-11-25T07:29:18,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741939_1119 (size=5153) 2024-11-25T07:29:18,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741939_1119 (size=5153) 2024-11-25T07:29:18,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741939_1119 (size=5153) 2024-11-25T07:29:18,833 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740/.tmp/ns/f34515f6c733485681bc7f747ca0223c 2024-11-25T07:29:18,852 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740/.tmp/table/444d7cff4dd7414196e9554715c2e913 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732519735675/Put/seqid=0 2024-11-25T07:29:18,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741940_1120 (size=5431) 2024-11-25T07:29:18,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741940_1120 (size=5431) 2024-11-25T07:29:18,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741940_1120 (size=5431) 2024-11-25T07:29:18,859 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740/.tmp/table/444d7cff4dd7414196e9554715c2e913 2024-11-25T07:29:18,865 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740/.tmp/info/21c0aa379bd04031807a9c2e276f5028 as hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740/info/21c0aa379bd04031807a9c2e276f5028 2024-11-25T07:29:18,869 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740/info/21c0aa379bd04031807a9c2e276f5028, entries=18, sequenceid=21, filesize=8.0 K 2024-11-25T07:29:18,870 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740/.tmp/ns/f34515f6c733485681bc7f747ca0223c as hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740/ns/f34515f6c733485681bc7f747ca0223c 2024-11-25T07:29:18,874 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740/ns/f34515f6c733485681bc7f747ca0223c, entries=2, sequenceid=21, filesize=5.0 K 2024-11-25T07:29:18,875 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740/.tmp/table/444d7cff4dd7414196e9554715c2e913 as hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740/table/444d7cff4dd7414196e9554715c2e913 2024-11-25T07:29:18,879 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740/table/444d7cff4dd7414196e9554715c2e913, entries=2, sequenceid=21, filesize=5.3 K 2024-11-25T07:29:18,880 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 515ms, sequenceid=21, compaction requested=false 2024-11-25T07:29:18,884 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-25T07:29:18,885 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T07:29:18,885 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T07:29:18,885 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732519758364Running coprocessor pre-close hooks at 1732519758364Disabling compacts and flushes for region at 1732519758364Disabling writes for close at 1732519758365 (+1 ms)Obtaining lock to block concurrent updates at 1732519758365Preparing flush snapshotting stores in 1588230740 at 1732519758365Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1732519758365Flushing stores of hbase:meta,,1.1588230740 at 1732519758367 (+2 ms)Flushing 1588230740/info: creating writer at 1732519758368 (+1 ms)Flushing 1588230740/info: appending metadata at 1732519758391 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732519758391Flushing 1588230740/ns: creating writer at 1732519758808 (+417 ms)Flushing 1588230740/ns: appending metadata at 1732519758823 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732519758823Flushing 1588230740/table: creating writer at 1732519758838 (+15 ms)Flushing 1588230740/table: appending metadata at 1732519758852 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732519758852Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f5092a0: reopening flushed file at 1732519758864 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@635a7540: reopening flushed file at 1732519758869 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e0a35d: reopening flushed file at 1732519758874 (+5 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 515ms, sequenceid=21, compaction requested=false at 1732519758880 (+6 ms)Writing region close event to WAL at 1732519758881 (+1 ms)Running coprocessor post-close hooks at 1732519758885 (+4 ms)Closed at 1732519758885 2024-11-25T07:29:18,885 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T07:29:18,965 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(976): stopping server 5eb3d201e8c9,33753,1732519720813; all regions closed. 2024-11-25T07:29:18,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741836_1012 (size=8899) 2024-11-25T07:29:18,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741836_1012 (size=8899) 2024-11-25T07:29:18,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741836_1012 (size=8899) 2024-11-25T07:29:18,970 DEBUG [RS:0;5eb3d201e8c9:33753 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/oldWALs 2024-11-25T07:29:18,971 INFO [RS:0;5eb3d201e8c9:33753 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5eb3d201e8c9%2C33753%2C1732519720813.meta:.meta(num 1732519722669) 2024-11-25T07:29:18,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741833_1009 (size=3433) 2024-11-25T07:29:18,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741833_1009 (size=3433) 2024-11-25T07:29:18,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741833_1009 (size=3433) 2024-11-25T07:29:18,975 DEBUG [RS:0;5eb3d201e8c9:33753 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/oldWALs 2024-11-25T07:29:18,975 INFO [RS:0;5eb3d201e8c9:33753 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5eb3d201e8c9%2C33753%2C1732519720813:(num 1732519722293) 2024-11-25T07:29:18,975 DEBUG [RS:0;5eb3d201e8c9:33753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:29:18,975 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T07:29:18,975 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T07:29:18,976 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.ChoreService(370): Chore service for: regionserver/5eb3d201e8c9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T07:29:18,976 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T07:29:18,976 INFO [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T07:29:18,976 INFO [RS:0;5eb3d201e8c9:33753 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33753 2024-11-25T07:29:18,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T07:29:18,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5eb3d201e8c9,33753,1732519720813 2024-11-25T07:29:18,978 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T07:29:18,979 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5eb3d201e8c9,33753,1732519720813] 2024-11-25T07:29:18,980 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5eb3d201e8c9,33753,1732519720813 already deleted, retry=false 2024-11-25T07:29:18,980 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5eb3d201e8c9,33753,1732519720813 expired; onlineServers=0 2024-11-25T07:29:18,980 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '5eb3d201e8c9,33903,1732519720118' ***** 2024-11-25T07:29:18,981 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T07:29:18,981 INFO [M:0;5eb3d201e8c9:33903 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T07:29:18,981 INFO [M:0;5eb3d201e8c9:33903 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T07:29:18,981 DEBUG [M:0;5eb3d201e8c9:33903 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T07:29:18,981 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T07:29:18,981 DEBUG [M:0;5eb3d201e8c9:33903 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T07:29:18,981 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster-HFileCleaner.large.0-1732519721933 {}] cleaner.HFileCleaner(306): Exit Thread[master/5eb3d201e8c9:0:becomeActiveMaster-HFileCleaner.large.0-1732519721933,5,FailOnTimeoutGroup] 2024-11-25T07:29:18,981 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster-HFileCleaner.small.0-1732519721934 {}] cleaner.HFileCleaner(306): Exit Thread[master/5eb3d201e8c9:0:becomeActiveMaster-HFileCleaner.small.0-1732519721934,5,FailOnTimeoutGroup] 2024-11-25T07:29:18,981 INFO [M:0;5eb3d201e8c9:33903 {}] hbase.ChoreService(370): Chore service for: master/5eb3d201e8c9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T07:29:18,981 INFO [M:0;5eb3d201e8c9:33903 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T07:29:18,981 DEBUG [M:0;5eb3d201e8c9:33903 {}] master.HMaster(1795): Stopping service threads 2024-11-25T07:29:18,982 INFO [M:0;5eb3d201e8c9:33903 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T07:29:18,982 INFO [M:0;5eb3d201e8c9:33903 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T07:29:18,982 INFO [M:0;5eb3d201e8c9:33903 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T07:29:18,982 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T07:29:18,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T07:29:18,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:29:18,984 DEBUG [M:0;5eb3d201e8c9:33903 {}] zookeeper.ZKUtil(347): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T07:29:18,984 WARN [M:0;5eb3d201e8c9:33903 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T07:29:18,985 INFO [M:0;5eb3d201e8c9:33903 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/.lastflushedseqids 2024-11-25T07:29:18,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741941_1121 (size=138) 2024-11-25T07:29:18,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741941_1121 (size=138) 2024-11-25T07:29:18,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741941_1121 (size=138) 2024-11-25T07:29:18,999 INFO [M:0;5eb3d201e8c9:33903 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T07:29:19,000 INFO [M:0;5eb3d201e8c9:33903 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T07:29:19,000 DEBUG [M:0;5eb3d201e8c9:33903 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T07:29:19,000 INFO [M:0;5eb3d201e8c9:33903 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T07:29:19,000 DEBUG [M:0;5eb3d201e8c9:33903 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T07:29:19,000 DEBUG [M:0;5eb3d201e8c9:33903 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T07:29:19,000 DEBUG [M:0;5eb3d201e8c9:33903 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T07:29:19,000 INFO [M:0;5eb3d201e8c9:33903 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.33 KB heapSize=83.72 KB 2024-11-25T07:29:19,016 DEBUG [M:0;5eb3d201e8c9:33903 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/221b3288571e42d58eef33dc28a5eada is 82, key is hbase:meta,,1/info:regioninfo/1732519722756/Put/seqid=0 2024-11-25T07:29:19,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741942_1122 (size=5672) 2024-11-25T07:29:19,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741942_1122 (size=5672) 2024-11-25T07:29:19,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741942_1122 (size=5672) 2024-11-25T07:29:19,023 INFO [M:0;5eb3d201e8c9:33903 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/221b3288571e42d58eef33dc28a5eada 2024-11-25T07:29:19,044 DEBUG [M:0;5eb3d201e8c9:33903 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ad1a955d006a463a8055b6b72d909388 is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732519735681/Put/seqid=0 2024-11-25T07:29:19,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741943_1123 (size=7754) 2024-11-25T07:29:19,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741943_1123 (size=7754) 2024-11-25T07:29:19,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741943_1123 (size=7754) 2024-11-25T07:29:19,051 INFO [M:0;5eb3d201e8c9:33903 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ad1a955d006a463a8055b6b72d909388 2024-11-25T07:29:19,056 INFO [M:0;5eb3d201e8c9:33903 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ad1a955d006a463a8055b6b72d909388 2024-11-25T07:29:19,071 DEBUG [M:0;5eb3d201e8c9:33903 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/711656602c3547b4a920684fa41ce54b is 69, key is 5eb3d201e8c9,33753,1732519720813/rs:state/1732519722039/Put/seqid=0 2024-11-25T07:29:19,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741944_1124 (size=5445) 2024-11-25T07:29:19,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741944_1124 (size=5445) 2024-11-25T07:29:19,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741944_1124 (size=5445) 2024-11-25T07:29:19,079 INFO [M:0;5eb3d201e8c9:33903 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/711656602c3547b4a920684fa41ce54b 2024-11-25T07:29:19,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T07:29:19,080 INFO [RS:0;5eb3d201e8c9:33753 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T07:29:19,080 INFO [RS:0;5eb3d201e8c9:33753 {}] regionserver.HRegionServer(1031): Exiting; stopping=5eb3d201e8c9,33753,1732519720813; zookeeper connection closed. 2024-11-25T07:29:19,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33753-0x1014e06dc560001, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T07:29:19,080 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@762822f7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@762822f7 2024-11-25T07:29:19,081 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-25T07:29:19,084 INFO [M:0;5eb3d201e8c9:33903 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 711656602c3547b4a920684fa41ce54b 2024-11-25T07:29:19,085 DEBUG [M:0;5eb3d201e8c9:33903 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/221b3288571e42d58eef33dc28a5eada as hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/221b3288571e42d58eef33dc28a5eada 2024-11-25T07:29:19,089 INFO [M:0;5eb3d201e8c9:33903 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/221b3288571e42d58eef33dc28a5eada, entries=8, sequenceid=168, filesize=5.5 K 2024-11-25T07:29:19,090 DEBUG [M:0;5eb3d201e8c9:33903 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ad1a955d006a463a8055b6b72d909388 as hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ad1a955d006a463a8055b6b72d909388 2024-11-25T07:29:19,094 INFO [M:0;5eb3d201e8c9:33903 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ad1a955d006a463a8055b6b72d909388 2024-11-25T07:29:19,094 INFO [M:0;5eb3d201e8c9:33903 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ad1a955d006a463a8055b6b72d909388, entries=17, sequenceid=168, filesize=7.6 K 2024-11-25T07:29:19,095 DEBUG [M:0;5eb3d201e8c9:33903 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/711656602c3547b4a920684fa41ce54b as hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/711656602c3547b4a920684fa41ce54b 2024-11-25T07:29:19,099 INFO [M:0;5eb3d201e8c9:33903 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 711656602c3547b4a920684fa41ce54b 2024-11-25T07:29:19,099 INFO [M:0;5eb3d201e8c9:33903 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44261/user/jenkins/test-data/6530b269-9419-98a6-01ef-ee2bf8b43438/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/711656602c3547b4a920684fa41ce54b, entries=3, sequenceid=168, filesize=5.3 K 2024-11-25T07:29:19,100 INFO [M:0;5eb3d201e8c9:33903 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.33 KB/69972, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 100ms, sequenceid=168, compaction requested=false 2024-11-25T07:29:19,101 INFO [M:0;5eb3d201e8c9:33903 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T07:29:19,101 DEBUG [M:0;5eb3d201e8c9:33903 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732519759000Disabling compacts and flushes for region at 1732519759000Disabling writes for close at 1732519759000Obtaining lock to block concurrent updates at 1732519759000Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732519759000Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69972, getHeapSize=85664, getOffHeapSize=0, getCellsCount=195 at 1732519759000Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732519759001 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732519759001Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732519759016 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732519759016Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732519759028 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732519759043 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732519759043Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732519759056 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732519759070 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732519759070Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b9bf904: reopening flushed file at 1732519759084 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@683f2c9c: reopening flushed file at 1732519759089 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38ee9584: reopening flushed file at 1732519759094 (+5 ms)Finished flush of dataSize ~68.33 KB/69972, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 100ms, sequenceid=168, compaction requested=false at 1732519759100 (+6 ms)Writing region close event to WAL at 1732519759101 (+1 ms)Closed at 1732519759101 2024-11-25T07:29:19,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37227 is added to blk_1073741830_1006 (size=81419) 2024-11-25T07:29:19,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33603 is added to blk_1073741830_1006 (size=81419) 2024-11-25T07:29:19,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37043 is added to blk_1073741830_1006 (size=81419) 2024-11-25T07:29:19,104 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T07:29:19,104 INFO [M:0;5eb3d201e8c9:33903 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T07:29:19,104 INFO [M:0;5eb3d201e8c9:33903 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33903 2024-11-25T07:29:19,105 INFO [M:0;5eb3d201e8c9:33903 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T07:29:19,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T07:29:19,207 INFO [M:0;5eb3d201e8c9:33903 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T07:29:19,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33903-0x1014e06dc560000, quorum=127.0.0.1:50797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T07:29:19,214 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738961 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738961 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1862095393_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732519738653/wal.1732519738961 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1862095393_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-25T07:29:19,215 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732519730856/wal.1732519730984 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:19,215 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732519753688/wal.1732519753945 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:19,218 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519748360 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519748360 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1862095393_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732519739114/wal.1732519748360 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1862095393_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-25T07:29:19,219 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732519723894/wal.1732519723963 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:19,221 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519749050 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519749050 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1862095393_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732519748563/wal.1732519749050 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1862095393_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-25T07:29:19,222 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732519749232/wal.1732519749304 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:29:19,225 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal.1732519723764 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal.1732519723764 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1862095393_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732519723585/wal.1732519723764 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1862095393_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-25T07:29:19,227 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal.1732519723339 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal.1732519723339 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1862095393_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732519723103/wal.1732519723339 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1862095393_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-25T07:29:19,231 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1c5b5cda{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T07:29:19,234 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@32eb964{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T07:29:19,234 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T07:29:19,234 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@582ba343{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T07:29:19,234 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7bd4c69c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/hadoop.log.dir/,STOPPED} 2024-11-25T07:29:19,237 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T07:29:19,237 WARN [BP-661403191-172.17.0.2-1732519717195 heartbeating to localhost/127.0.0.1:44261 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T07:29:19,238 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T07:29:19,238 WARN [BP-661403191-172.17.0.2-1732519717195 heartbeating to localhost/127.0.0.1:44261 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-661403191-172.17.0.2-1732519717195 (Datanode Uuid f75da14a-acc5-4dbe-9081-d37c01774a00) service to localhost/127.0.0.1:44261 2024-11-25T07:29:19,239 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data5/current/BP-661403191-172.17.0.2-1732519717195 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T07:29:19,239 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data6/current/BP-661403191-172.17.0.2-1732519717195 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T07:29:19,240 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T07:29:19,242 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@70b25848{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T07:29:19,242 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6a8a49ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T07:29:19,242 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T07:29:19,242 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4edd15fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T07:29:19,243 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b152b3a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/hadoop.log.dir/,STOPPED} 2024-11-25T07:29:19,244 WARN [BP-661403191-172.17.0.2-1732519717195 heartbeating to localhost/127.0.0.1:44261 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T07:29:19,244 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T07:29:19,244 WARN [BP-661403191-172.17.0.2-1732519717195 heartbeating to localhost/127.0.0.1:44261 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-661403191-172.17.0.2-1732519717195 (Datanode Uuid 48ca7e09-c38c-495c-bc02-27c692a838eb) service to localhost/127.0.0.1:44261 2024-11-25T07:29:19,244 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T07:29:19,244 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data3/current/BP-661403191-172.17.0.2-1732519717195 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T07:29:19,245 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data4/current/BP-661403191-172.17.0.2-1732519717195 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T07:29:19,245 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T07:29:19,254 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@17be2a4f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T07:29:19,255 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2ecf726e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T07:29:19,255 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T07:29:19,255 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@19895485{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T07:29:19,255 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@50901468{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/hadoop.log.dir/,STOPPED} 2024-11-25T07:29:19,256 WARN [BP-661403191-172.17.0.2-1732519717195 heartbeating to localhost/127.0.0.1:44261 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T07:29:19,256 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T07:29:19,256 WARN [BP-661403191-172.17.0.2-1732519717195 heartbeating to localhost/127.0.0.1:44261 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-661403191-172.17.0.2-1732519717195 (Datanode Uuid c7b23268-a363-4476-9499-b348c306f2b9) service to localhost/127.0.0.1:44261 2024-11-25T07:29:19,256 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T07:29:19,257 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data1/current/BP-661403191-172.17.0.2-1732519717195 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T07:29:19,258 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/cluster_3f51eeaf-ca9d-054f-0779-c2d5935a4d65/data/data2/current/BP-661403191-172.17.0.2-1732519717195 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T07:29:19,258 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T07:29:19,264 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@73b32aa2{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T07:29:19,265 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e2ddae{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T07:29:19,265 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T07:29:19,265 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7e3a0561{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T07:29:19,265 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751489f9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f48dfa8-7a7a-2264-00db-35e140873d00/hadoop.log.dir/,STOPPED} 2024-11-25T07:29:19,275 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T07:29:19,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down