2024-11-23 18:49:18,416 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-23 18:49:18,429 main DEBUG Took 0.010273 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-23 18:49:18,429 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-23 18:49:18,429 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-23 18:49:18,431 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-23 18:49:18,432 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:49:18,438 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-23 18:49:18,453 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:49:18,454 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:49:18,454 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:49:18,455 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:49:18,455 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:49:18,455 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:49:18,456 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:49:18,457 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:49:18,457 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:49:18,458 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:49:18,459 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:49:18,459 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:49:18,459 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:49:18,460 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:49:18,460 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:49:18,461 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:49:18,461 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:49:18,461 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:49:18,462 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:49:18,462 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:49:18,462 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:49:18,463 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:49:18,463 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:49:18,463 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:49:18,464 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:49:18,464 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-23 18:49:18,466 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:49:18,468 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-23 18:49:18,470 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-23 18:49:18,471 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-23 18:49:18,472 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-23 18:49:18,473 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-23 18:49:18,481 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-23 18:49:18,484 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-23 18:49:18,485 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-23 18:49:18,486 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-23 18:49:18,486 main DEBUG createAppenders(={Console}) 2024-11-23 18:49:18,487 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-23 18:49:18,487 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-23 18:49:18,487 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-23 18:49:18,488 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-23 18:49:18,488 main DEBUG OutputStream closed 2024-11-23 18:49:18,489 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-23 18:49:18,489 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-23 18:49:18,489 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-23 18:49:18,572 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-23 18:49:18,574 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-23 18:49:18,575 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-23 18:49:18,576 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-23 18:49:18,576 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-23 18:49:18,577 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-23 18:49:18,577 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-23 18:49:18,578 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-23 18:49:18,578 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-23 18:49:18,578 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-23 18:49:18,579 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-23 18:49:18,579 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-23 18:49:18,580 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-23 18:49:18,580 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-23 18:49:18,581 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-23 18:49:18,581 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-23 18:49:18,581 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-23 18:49:18,583 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-23 18:49:18,585 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23 18:49:18,585 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-23 18:49:18,585 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-23 18:49:18,586 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-23T18:49:18,825 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef 2024-11-23 18:49:18,828 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-23 18:49:18,828 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23T18:49:18,837 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-23T18:49:18,844 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayValueCompression timeout: 13 mins 2024-11-23T18:49:18,867 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-23T18:49:18,912 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-23T18:49:18,912 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-23T18:49:18,924 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-23T18:49:18,938 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789, deleteOnExit=true 2024-11-23T18:49:18,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-23T18:49:18,940 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/test.cache.data in system properties and HBase conf 2024-11-23T18:49:18,940 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/hadoop.tmp.dir in system properties and HBase conf 2024-11-23T18:49:18,941 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/hadoop.log.dir in system properties and HBase conf 2024-11-23T18:49:18,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-23T18:49:18,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-23T18:49:18,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-23T18:49:19,018 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-23T18:49:19,111 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T18:49:19,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-23T18:49:19,117 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-23T18:49:19,118 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-23T18:49:19,118 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T18:49:19,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-23T18:49:19,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-23T18:49:19,120 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T18:49:19,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T18:49:19,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-23T18:49:19,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/nfs.dump.dir in system properties and HBase conf 2024-11-23T18:49:19,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/java.io.tmpdir in system properties and HBase conf 2024-11-23T18:49:19,123 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T18:49:19,123 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-23T18:49:19,124 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-23T18:49:20,183 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-23T18:49:20,258 INFO [Time-limited test {}] log.Log(170): Logging initialized @2551ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-23T18:49:20,326 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:49:20,391 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:49:20,415 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:49:20,415 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:49:20,417 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T18:49:20,433 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:49:20,437 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:49:20,440 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:49:20,593 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3717288f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/java.io.tmpdir/jetty-localhost-46323-hadoop-hdfs-3_4_1-tests_jar-_-any-15574649999456909008/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T18:49:20,598 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:46323} 2024-11-23T18:49:20,598 INFO [Time-limited test {}] server.Server(415): Started @2890ms 2024-11-23T18:49:21,161 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:49:21,167 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:49:21,168 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:49:21,168 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:49:21,169 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T18:49:21,169 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:49:21,170 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:49:21,266 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@330740de{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/java.io.tmpdir/jetty-localhost-39829-hadoop-hdfs-3_4_1-tests_jar-_-any-8956015444358566003/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:49:21,267 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:39829} 2024-11-23T18:49:21,267 INFO [Time-limited test {}] server.Server(415): Started @3560ms 2024-11-23T18:49:21,310 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:49:21,414 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:49:21,421 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:49:21,422 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:49:21,423 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:49:21,423 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T18:49:21,424 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:49:21,425 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:49:21,553 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bd427b8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/java.io.tmpdir/jetty-localhost-35997-hadoop-hdfs-3_4_1-tests_jar-_-any-4099582368970429070/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:49:21,554 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:35997} 2024-11-23T18:49:21,554 INFO [Time-limited test {}] server.Server(415): Started @3847ms 2024-11-23T18:49:21,556 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:49:21,591 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:49:21,596 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:49:21,597 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:49:21,597 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:49:21,598 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T18:49:21,598 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:49:21,599 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:49:21,694 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@35f1150e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/java.io.tmpdir/jetty-localhost-34045-hadoop-hdfs-3_4_1-tests_jar-_-any-4629245865102276396/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:49:21,695 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:34045} 2024-11-23T18:49:21,695 INFO [Time-limited test {}] server.Server(415): Started @3988ms 2024-11-23T18:49:21,697 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:49:23,186 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/data/data2/current/BP-995492633-172.17.0.2-1732387759654/current, will proceed with Du for space computation calculation, 2024-11-23T18:49:23,186 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/data/data3/current/BP-995492633-172.17.0.2-1732387759654/current, will proceed with Du for space computation calculation, 2024-11-23T18:49:23,186 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/data/data4/current/BP-995492633-172.17.0.2-1732387759654/current, will proceed with Du for space computation calculation, 2024-11-23T18:49:23,186 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/data/data1/current/BP-995492633-172.17.0.2-1732387759654/current, will proceed with Du for space computation calculation, 2024-11-23T18:49:23,220 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:49:23,220 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:49:23,264 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcb29fad7b9e5c4d2 with lease ID 0xacbe2a905438fb0e: Processing first storage report for DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18 from datanode DatanodeRegistration(127.0.0.1:43523, datanodeUuid=fd34b4b4-85d3-4d1a-8085-dd5cabccc71e, infoPort=43381, infoSecurePort=0, ipcPort=35915, storageInfo=lv=-57;cid=testClusterID;nsid=1735614142;c=1732387759654) 2024-11-23T18:49:23,265 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcb29fad7b9e5c4d2 with lease ID 0xacbe2a905438fb0e: from storage DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18 node DatanodeRegistration(127.0.0.1:43523, datanodeUuid=fd34b4b4-85d3-4d1a-8085-dd5cabccc71e, infoPort=43381, infoSecurePort=0, ipcPort=35915, storageInfo=lv=-57;cid=testClusterID;nsid=1735614142;c=1732387759654), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-23T18:49:23,266 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe1d5964f6703c8e8 with lease ID 0xacbe2a905438fb0f: Processing first storage report for DS-99681bba-29f3-46cb-9183-ad1978fc6996 from datanode DatanodeRegistration(127.0.0.1:40877, datanodeUuid=717b7c20-a1ca-4fc2-903e-6487ea215d67, infoPort=39619, infoSecurePort=0, ipcPort=34429, storageInfo=lv=-57;cid=testClusterID;nsid=1735614142;c=1732387759654) 2024-11-23T18:49:23,266 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe1d5964f6703c8e8 with lease ID 0xacbe2a905438fb0f: from storage DS-99681bba-29f3-46cb-9183-ad1978fc6996 node DatanodeRegistration(127.0.0.1:40877, datanodeUuid=717b7c20-a1ca-4fc2-903e-6487ea215d67, infoPort=39619, infoSecurePort=0, ipcPort=34429, storageInfo=lv=-57;cid=testClusterID;nsid=1735614142;c=1732387759654), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-23T18:49:23,266 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcb29fad7b9e5c4d2 with lease ID 0xacbe2a905438fb0e: Processing first storage report for DS-bf096297-1e10-4276-a946-f3e0ec32beeb from datanode DatanodeRegistration(127.0.0.1:43523, datanodeUuid=fd34b4b4-85d3-4d1a-8085-dd5cabccc71e, infoPort=43381, infoSecurePort=0, ipcPort=35915, storageInfo=lv=-57;cid=testClusterID;nsid=1735614142;c=1732387759654) 2024-11-23T18:49:23,266 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcb29fad7b9e5c4d2 with lease ID 0xacbe2a905438fb0e: from storage DS-bf096297-1e10-4276-a946-f3e0ec32beeb node DatanodeRegistration(127.0.0.1:43523, datanodeUuid=fd34b4b4-85d3-4d1a-8085-dd5cabccc71e, infoPort=43381, infoSecurePort=0, ipcPort=35915, storageInfo=lv=-57;cid=testClusterID;nsid=1735614142;c=1732387759654), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:49:23,267 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe1d5964f6703c8e8 with lease ID 0xacbe2a905438fb0f: Processing first storage report for DS-344ec07c-af1d-4687-90bb-43ad4200fb85 from datanode DatanodeRegistration(127.0.0.1:40877, datanodeUuid=717b7c20-a1ca-4fc2-903e-6487ea215d67, infoPort=39619, infoSecurePort=0, ipcPort=34429, storageInfo=lv=-57;cid=testClusterID;nsid=1735614142;c=1732387759654) 2024-11-23T18:49:23,267 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe1d5964f6703c8e8 with lease ID 0xacbe2a905438fb0f: from storage DS-344ec07c-af1d-4687-90bb-43ad4200fb85 node DatanodeRegistration(127.0.0.1:40877, datanodeUuid=717b7c20-a1ca-4fc2-903e-6487ea215d67, infoPort=39619, infoSecurePort=0, ipcPort=34429, storageInfo=lv=-57;cid=testClusterID;nsid=1735614142;c=1732387759654), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:49:23,294 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/data/data6/current/BP-995492633-172.17.0.2-1732387759654/current, will proceed with Du for space computation calculation, 2024-11-23T18:49:23,294 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/data/data5/current/BP-995492633-172.17.0.2-1732387759654/current, will proceed with Du for space computation calculation, 2024-11-23T18:49:23,315 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:49:23,319 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd271899c263726b with lease ID 0xacbe2a905438fb10: Processing first storage report for DS-4bd7ba11-1981-4abe-8a12-566437cf080c from datanode DatanodeRegistration(127.0.0.1:37657, datanodeUuid=3e540eaa-9a8b-4e0d-ba02-a2f0c9380641, infoPort=37451, infoSecurePort=0, ipcPort=45387, storageInfo=lv=-57;cid=testClusterID;nsid=1735614142;c=1732387759654) 2024-11-23T18:49:23,319 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd271899c263726b with lease ID 0xacbe2a905438fb10: from storage DS-4bd7ba11-1981-4abe-8a12-566437cf080c node DatanodeRegistration(127.0.0.1:37657, datanodeUuid=3e540eaa-9a8b-4e0d-ba02-a2f0c9380641, infoPort=37451, infoSecurePort=0, ipcPort=45387, storageInfo=lv=-57;cid=testClusterID;nsid=1735614142;c=1732387759654), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-23T18:49:23,319 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd271899c263726b with lease ID 0xacbe2a905438fb10: Processing first storage report for DS-15bfd7c8-b16d-4276-86e8-680236bb1e15 from datanode DatanodeRegistration(127.0.0.1:37657, datanodeUuid=3e540eaa-9a8b-4e0d-ba02-a2f0c9380641, infoPort=37451, infoSecurePort=0, ipcPort=45387, storageInfo=lv=-57;cid=testClusterID;nsid=1735614142;c=1732387759654) 2024-11-23T18:49:23,319 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd271899c263726b with lease ID 0xacbe2a905438fb10: from storage DS-15bfd7c8-b16d-4276-86e8-680236bb1e15 node DatanodeRegistration(127.0.0.1:37657, datanodeUuid=3e540eaa-9a8b-4e0d-ba02-a2f0c9380641, infoPort=37451, infoSecurePort=0, ipcPort=45387, storageInfo=lv=-57;cid=testClusterID;nsid=1735614142;c=1732387759654), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:49:23,426 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef 2024-11-23T18:49:23,489 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/zookeeper_0, clientPort=57018, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T18:49:23,511 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57018 2024-11-23T18:49:23,527 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:49:23,529 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:49:23,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741825_1001 (size=7) 2024-11-23T18:49:23,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741825_1001 (size=7) 2024-11-23T18:49:23,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741825_1001 (size=7) 2024-11-23T18:49:24,122 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe with version=8 2024-11-23T18:49:24,123 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/hbase-staging 2024-11-23T18:49:24,409 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:49:24,417 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:49:24,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:49:24,422 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:49:24,422 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:49:24,422 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:49:24,547 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T18:49:24,601 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-23T18:49:24,611 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-23T18:49:24,614 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:49:24,638 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 62746 (auto-detected) 2024-11-23T18:49:24,639 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-23T18:49:24,656 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37117 2024-11-23T18:49:24,674 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37117 connecting to ZooKeeper ensemble=127.0.0.1:57018 2024-11-23T18:49:24,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:371170x0, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:49:24,801 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37117-0x101690770ba0000 connected 2024-11-23T18:49:24,893 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:49:24,895 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:49:24,904 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:49:24,908 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe, hbase.cluster.distributed=false 2024-11-23T18:49:24,928 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:49:24,932 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37117 2024-11-23T18:49:24,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37117 2024-11-23T18:49:24,937 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37117 2024-11-23T18:49:24,937 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37117 2024-11-23T18:49:24,938 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37117 2024-11-23T18:49:25,030 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:49:25,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:49:25,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:49:25,032 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:49:25,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:49:25,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:49:25,035 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T18:49:25,037 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:49:25,038 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32913 2024-11-23T18:49:25,040 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32913 connecting to ZooKeeper ensemble=127.0.0.1:57018 2024-11-23T18:49:25,041 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:49:25,045 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:49:25,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:329130x0, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:49:25,061 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:329130x0, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:49:25,061 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32913-0x101690770ba0001 connected 2024-11-23T18:49:25,064 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T18:49:25,071 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T18:49:25,074 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32913-0x101690770ba0001, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T18:49:25,081 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32913-0x101690770ba0001, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:49:25,081 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32913 2024-11-23T18:49:25,082 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32913 2024-11-23T18:49:25,082 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32913 2024-11-23T18:49:25,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32913 2024-11-23T18:49:25,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32913 2024-11-23T18:49:25,100 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:49:25,100 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:49:25,100 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:49:25,101 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:49:25,101 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:49:25,101 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:49:25,101 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T18:49:25,102 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:49:25,102 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43797 2024-11-23T18:49:25,104 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43797 connecting to ZooKeeper ensemble=127.0.0.1:57018 2024-11-23T18:49:25,105 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:49:25,108 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:49:25,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:437970x0, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:49:25,124 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43797-0x101690770ba0002 connected 2024-11-23T18:49:25,124 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:437970x0, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:49:25,125 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T18:49:25,126 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T18:49:25,127 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T18:49:25,129 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:49:25,130 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43797 2024-11-23T18:49:25,130 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43797 2024-11-23T18:49:25,130 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43797 2024-11-23T18:49:25,131 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43797 2024-11-23T18:49:25,131 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43797 2024-11-23T18:49:25,147 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:49:25,148 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:49:25,148 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:49:25,148 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:49:25,148 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:49:25,148 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:49:25,148 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T18:49:25,149 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:49:25,150 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34841 2024-11-23T18:49:25,151 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34841 connecting to ZooKeeper ensemble=127.0.0.1:57018 2024-11-23T18:49:25,152 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:49:25,155 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:49:25,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:348410x0, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:49:25,166 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:348410x0, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:49:25,166 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34841-0x101690770ba0003 connected 2024-11-23T18:49:25,166 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T18:49:25,167 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T18:49:25,168 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T18:49:25,170 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:49:25,171 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34841 2024-11-23T18:49:25,172 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34841 2024-11-23T18:49:25,172 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34841 2024-11-23T18:49:25,173 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34841 2024-11-23T18:49:25,173 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34841 2024-11-23T18:49:25,188 DEBUG [M:0;2eda60943ea1:37117 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2eda60943ea1:37117 2024-11-23T18:49:25,189 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2eda60943ea1,37117,1732387764258 2024-11-23T18:49:25,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:49:25,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32913-0x101690770ba0001, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:49:25,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:49:25,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:49:25,209 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2eda60943ea1,37117,1732387764258 2024-11-23T18:49:25,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T18:49:25,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T18:49:25,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32913-0x101690770ba0001, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T18:49:25,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:25,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:25,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32913-0x101690770ba0001, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:25,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:25,240 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T18:49:25,241 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2eda60943ea1,37117,1732387764258 from backup master directory 2024-11-23T18:49:25,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32913-0x101690770ba0001, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:49:25,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:49:25,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:49:25,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2eda60943ea1,37117,1732387764258 2024-11-23T18:49:25,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:49:25,250 WARN [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:49:25,251 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2eda60943ea1,37117,1732387764258 2024-11-23T18:49:25,253 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-23T18:49:25,255 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-23T18:49:25,307 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/hbase.id] with ID: c7282851-006a-445f-a51c-1e6fe6ae5ccd 2024-11-23T18:49:25,307 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/.tmp/hbase.id 2024-11-23T18:49:25,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741826_1002 (size=42) 2024-11-23T18:49:25,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741826_1002 (size=42) 2024-11-23T18:49:25,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741826_1002 (size=42) 2024-11-23T18:49:25,324 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/.tmp/hbase.id]:[hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/hbase.id] 2024-11-23T18:49:25,366 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:49:25,370 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T18:49:25,389 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-23T18:49:25,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32913-0x101690770ba0001, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:25,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:25,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:25,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:25,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741827_1003 (size=196) 2024-11-23T18:49:25,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741827_1003 (size=196) 2024-11-23T18:49:25,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741827_1003 (size=196) 2024-11-23T18:49:25,438 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T18:49:25,439 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T18:49:25,444 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:25,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741828_1004 (size=1189) 2024-11-23T18:49:25,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741828_1004 (size=1189) 2024-11-23T18:49:25,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741828_1004 (size=1189) 2024-11-23T18:49:25,491 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store 2024-11-23T18:49:25,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741829_1005 (size=34) 2024-11-23T18:49:25,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741829_1005 (size=34) 2024-11-23T18:49:25,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741829_1005 (size=34) 2024-11-23T18:49:25,517 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-23T18:49:25,520 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:25,521 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T18:49:25,521 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:49:25,521 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:49:25,523 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T18:49:25,523 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:49:25,523 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:49:25,524 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732387765521Disabling compacts and flushes for region at 1732387765521Disabling writes for close at 1732387765523 (+2 ms)Writing region close event to WAL at 1732387765523Closed at 1732387765523 2024-11-23T18:49:25,526 WARN [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/.initializing 2024-11-23T18:49:25,526 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/WALs/2eda60943ea1,37117,1732387764258 2024-11-23T18:49:25,533 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:49:25,547 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C37117%2C1732387764258, suffix=, logDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/WALs/2eda60943ea1,37117,1732387764258, archiveDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/oldWALs, maxLogs=10 2024-11-23T18:49:25,576 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/WALs/2eda60943ea1,37117,1732387764258/2eda60943ea1%2C37117%2C1732387764258.1732387765552, exclude list is [], retry=0 2024-11-23T18:49:25,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:25,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:25,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:25,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:25,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-23T18:49:25,635 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/WALs/2eda60943ea1,37117,1732387764258/2eda60943ea1%2C37117%2C1732387764258.1732387765552 2024-11-23T18:49:25,637 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:39619:39619)] 2024-11-23T18:49:25,637 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:25,637 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:25,640 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:49:25,641 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:49:25,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:49:25,696 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T18:49:25,698 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:25,701 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:49:25,701 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:49:25,705 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T18:49:25,705 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:25,706 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:25,706 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:49:25,708 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T18:49:25,709 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:25,709 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:25,710 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:49:25,713 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T18:49:25,713 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:25,714 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:25,715 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:49:25,719 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:49:25,720 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:49:25,725 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:49:25,726 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:49:25,729 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T18:49:25,732 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:49:25,736 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:49:25,738 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72829982, jitterRate=0.08525130152702332}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T18:49:25,746 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732387765652Initializing all the Stores at 1732387765654 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387765654Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387765655 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387765655Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387765656 (+1 ms)Cleaning up temporary data from old regions at 1732387765726 (+70 ms)Region opened successfully at 1732387765746 (+20 ms) 2024-11-23T18:49:25,747 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T18:49:25,780 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16eb22c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:49:25,808 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T18:49:25,817 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T18:49:25,817 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T18:49:25,820 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T18:49:25,822 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-23T18:49:25,826 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-23T18:49:25,826 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T18:49:25,854 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T18:49:25,862 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T18:49:25,909 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T18:49:25,914 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T18:49:25,917 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T18:49:25,930 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T18:49:25,932 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T18:49:25,937 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T18:49:25,943 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T18:49:25,945 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T18:49:25,954 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T18:49:25,979 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T18:49:25,986 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T18:49:25,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:49:25,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32913-0x101690770ba0001, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:49:25,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:49:25,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:49:25,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:25,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32913-0x101690770ba0001, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:25,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:25,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:26,002 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2eda60943ea1,37117,1732387764258, sessionid=0x101690770ba0000, setting cluster-up flag (Was=false) 2024-11-23T18:49:26,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:26,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:26,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:26,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32913-0x101690770ba0001, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:26,070 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T18:49:26,075 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2eda60943ea1,37117,1732387764258 2024-11-23T18:49:26,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:26,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:26,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:26,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32913-0x101690770ba0001, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:26,134 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T18:49:26,138 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2eda60943ea1,37117,1732387764258 2024-11-23T18:49:26,148 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T18:49:26,176 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer(746): ClusterId : c7282851-006a-445f-a51c-1e6fe6ae5ccd 2024-11-23T18:49:26,176 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.HRegionServer(746): ClusterId : c7282851-006a-445f-a51c-1e6fe6ae5ccd 2024-11-23T18:49:26,176 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.HRegionServer(746): ClusterId : c7282851-006a-445f-a51c-1e6fe6ae5ccd 2024-11-23T18:49:26,179 DEBUG [RS:2;2eda60943ea1:34841 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T18:49:26,179 DEBUG [RS:0;2eda60943ea1:32913 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T18:49:26,179 DEBUG [RS:1;2eda60943ea1:43797 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T18:49:26,205 DEBUG [RS:0;2eda60943ea1:32913 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T18:49:26,205 DEBUG [RS:2;2eda60943ea1:34841 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T18:49:26,205 DEBUG [RS:1;2eda60943ea1:43797 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T18:49:26,205 DEBUG [RS:0;2eda60943ea1:32913 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T18:49:26,205 DEBUG [RS:2;2eda60943ea1:34841 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T18:49:26,205 DEBUG [RS:1;2eda60943ea1:43797 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T18:49:26,223 INFO [AsyncFSWAL-0-hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData-prefix:2eda60943ea1,37117,1732387764258 {}] compress.Compression(560): Loaded codec org.apache.hadoop.hbase.io.compress.ReusableStreamGzipCodec for compression algorithm GZ 2024-11-23T18:49:26,229 DEBUG [RS:2;2eda60943ea1:34841 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T18:49:26,229 DEBUG [RS:0;2eda60943ea1:32913 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T18:49:26,229 DEBUG [RS:1;2eda60943ea1:43797 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T18:49:26,229 DEBUG [RS:1;2eda60943ea1:43797 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a55643b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:49:26,229 DEBUG [RS:0;2eda60943ea1:32913 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e12b743, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:49:26,229 DEBUG [RS:2;2eda60943ea1:34841 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60b94f05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:49:26,239 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T18:49:26,244 DEBUG [RS:0;2eda60943ea1:32913 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2eda60943ea1:32913 2024-11-23T18:49:26,244 DEBUG [RS:2;2eda60943ea1:34841 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;2eda60943ea1:34841 2024-11-23T18:49:26,247 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T18:49:26,247 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T18:49:26,247 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T18:49:26,247 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T18:49:26,247 DEBUG [RS:2;2eda60943ea1:34841 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T18:49:26,247 DEBUG [RS:0;2eda60943ea1:32913 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T18:49:26,247 DEBUG [RS:1;2eda60943ea1:43797 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2eda60943ea1:43797 2024-11-23T18:49:26,248 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T18:49:26,248 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T18:49:26,248 DEBUG [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T18:49:26,250 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.HRegionServer(2659): reportForDuty to master=2eda60943ea1,37117,1732387764258 with port=34841, startcode=1732387765147 2024-11-23T18:49:26,250 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.HRegionServer(2659): reportForDuty to master=2eda60943ea1,37117,1732387764258 with port=32913, startcode=1732387765000 2024-11-23T18:49:26,250 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer(2659): reportForDuty to master=2eda60943ea1,37117,1732387764258 with port=43797, startcode=1732387765099 2024-11-23T18:49:26,250 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T18:49:26,257 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T18:49:26,261 DEBUG [RS:2;2eda60943ea1:34841 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T18:49:26,261 DEBUG [RS:1;2eda60943ea1:43797 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T18:49:26,261 DEBUG [RS:0;2eda60943ea1:32913 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T18:49:26,263 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2eda60943ea1,37117,1732387764258 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T18:49:26,269 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:49:26,270 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:49:26,270 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:49:26,270 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:49:26,270 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2eda60943ea1:0, corePoolSize=10, maxPoolSize=10 2024-11-23T18:49:26,270 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,270 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:49:26,270 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,273 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732387796273 2024-11-23T18:49:26,275 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T18:49:26,276 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T18:49:26,280 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T18:49:26,280 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T18:49:26,281 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T18:49:26,281 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T18:49:26,284 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,288 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T18:49:26,288 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:49:26,289 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T18:49:26,290 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T18:49:26,290 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T18:49:26,293 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T18:49:26,294 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T18:49:26,295 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37469, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T18:49:26,295 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53313, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T18:49:26,295 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47245, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T18:49:26,296 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387766295,5,FailOnTimeoutGroup] 2024-11-23T18:49:26,296 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387766296,5,FailOnTimeoutGroup] 2024-11-23T18:49:26,297 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,297 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T18:49:26,297 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:26,298 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,298 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,298 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T18:49:26,300 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37117 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2eda60943ea1,34841,1732387765147 2024-11-23T18:49:26,303 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37117 {}] master.ServerManager(517): Registering regionserver=2eda60943ea1,34841,1732387765147 2024-11-23T18:49:26,329 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37117 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2eda60943ea1,32913,1732387765000 2024-11-23T18:49:26,329 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37117 {}] master.ServerManager(517): Registering regionserver=2eda60943ea1,32913,1732387765000 2024-11-23T18:49:26,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741831_1007 (size=1321) 2024-11-23T18:49:26,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741831_1007 (size=1321) 2024-11-23T18:49:26,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741831_1007 (size=1321) 2024-11-23T18:49:26,333 DEBUG [RS:2;2eda60943ea1:34841 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe 2024-11-23T18:49:26,334 DEBUG [RS:2;2eda60943ea1:34841 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43835 2024-11-23T18:49:26,334 DEBUG [RS:2;2eda60943ea1:34841 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T18:49:26,335 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T18:49:26,335 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37117 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2eda60943ea1,43797,1732387765099 2024-11-23T18:49:26,335 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37117 {}] master.ServerManager(517): Registering regionserver=2eda60943ea1,43797,1732387765099 2024-11-23T18:49:26,335 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe 2024-11-23T18:49:26,335 DEBUG [RS:0;2eda60943ea1:32913 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe 2024-11-23T18:49:26,335 DEBUG [RS:0;2eda60943ea1:32913 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43835 2024-11-23T18:49:26,335 DEBUG [RS:0;2eda60943ea1:32913 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T18:49:26,339 DEBUG [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe 2024-11-23T18:49:26,339 DEBUG [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43835 2024-11-23T18:49:26,339 DEBUG [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T18:49:26,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741832_1008 (size=32) 2024-11-23T18:49:26,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741832_1008 (size=32) 2024-11-23T18:49:26,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741832_1008 (size=32) 2024-11-23T18:49:26,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:26,354 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T18:49:26,357 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T18:49:26,357 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:26,358 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:49:26,358 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T18:49:26,360 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T18:49:26,360 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:26,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:49:26,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T18:49:26,364 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T18:49:26,364 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:26,365 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:49:26,365 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T18:49:26,368 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T18:49:26,368 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:26,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:49:26,369 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T18:49:26,370 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740 2024-11-23T18:49:26,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740 2024-11-23T18:49:26,373 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T18:49:26,374 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T18:49:26,375 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T18:49:26,377 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T18:49:26,380 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:49:26,381 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70991930, jitterRate=0.05786219239234924}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T18:49:26,385 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732387766352Initializing all the Stores at 1732387766354 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387766354Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387766354Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387766354Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387766354Cleaning up temporary data from old regions at 1732387766374 (+20 ms)Region opened successfully at 1732387766385 (+11 ms) 2024-11-23T18:49:26,385 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T18:49:26,385 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T18:49:26,385 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T18:49:26,385 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T18:49:26,385 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T18:49:26,387 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T18:49:26,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732387766385Disabling compacts and flushes for region at 1732387766385Disabling writes for close at 1732387766385Writing region close event to WAL at 1732387766386 (+1 ms)Closed at 1732387766387 (+1 ms) 2024-11-23T18:49:26,390 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:49:26,390 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T18:49:26,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T18:49:26,404 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:49:26,407 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T18:49:26,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:49:26,470 DEBUG [RS:2;2eda60943ea1:34841 {}] zookeeper.ZKUtil(111): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2eda60943ea1,34841,1732387765147 2024-11-23T18:49:26,470 DEBUG [RS:0;2eda60943ea1:32913 {}] zookeeper.ZKUtil(111): regionserver:32913-0x101690770ba0001, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2eda60943ea1,32913,1732387765000 2024-11-23T18:49:26,471 WARN [RS:0;2eda60943ea1:32913 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:49:26,471 WARN [RS:2;2eda60943ea1:34841 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:49:26,471 INFO [RS:2;2eda60943ea1:34841 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:26,471 INFO [RS:0;2eda60943ea1:32913 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:26,471 DEBUG [RS:1;2eda60943ea1:43797 {}] zookeeper.ZKUtil(111): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2eda60943ea1,43797,1732387765099 2024-11-23T18:49:26,471 DEBUG [RS:0;2eda60943ea1:32913 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000 2024-11-23T18:49:26,471 DEBUG [RS:2;2eda60943ea1:34841 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,34841,1732387765147 2024-11-23T18:49:26,471 WARN [RS:1;2eda60943ea1:43797 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:49:26,471 INFO [RS:1;2eda60943ea1:43797 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:26,471 DEBUG [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,43797,1732387765099 2024-11-23T18:49:26,472 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2eda60943ea1,34841,1732387765147] 2024-11-23T18:49:26,472 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2eda60943ea1,32913,1732387765000] 2024-11-23T18:49:26,472 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2eda60943ea1,43797,1732387765099] 2024-11-23T18:49:26,496 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T18:49:26,496 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T18:49:26,496 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T18:49:26,511 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T18:49:26,511 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T18:49:26,511 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T18:49:26,517 INFO [RS:0;2eda60943ea1:32913 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T18:49:26,517 INFO [RS:1;2eda60943ea1:43797 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T18:49:26,517 INFO [RS:2;2eda60943ea1:34841 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T18:49:26,517 INFO [RS:1;2eda60943ea1:43797 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,517 INFO [RS:0;2eda60943ea1:32913 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,517 INFO [RS:2;2eda60943ea1:34841 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,518 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T18:49:26,518 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T18:49:26,519 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T18:49:26,524 INFO [RS:2;2eda60943ea1:34841 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T18:49:26,524 INFO [RS:1;2eda60943ea1:43797 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T18:49:26,524 INFO [RS:0;2eda60943ea1:32913 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T18:49:26,526 INFO [RS:1;2eda60943ea1:43797 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,526 INFO [RS:0;2eda60943ea1:32913 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,526 INFO [RS:2;2eda60943ea1:34841 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,526 DEBUG [RS:1;2eda60943ea1:43797 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,526 DEBUG [RS:0;2eda60943ea1:32913 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,526 DEBUG [RS:2;2eda60943ea1:34841 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,526 DEBUG [RS:2;2eda60943ea1:34841 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,526 DEBUG [RS:1;2eda60943ea1:43797 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,526 DEBUG [RS:0;2eda60943ea1:32913 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:2;2eda60943ea1:34841 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:1;2eda60943ea1:43797 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:0;2eda60943ea1:32913 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:2;2eda60943ea1:34841 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:2;2eda60943ea1:34841 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:0;2eda60943ea1:32913 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:1;2eda60943ea1:43797 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:2;2eda60943ea1:34841 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:49:26,527 DEBUG [RS:0;2eda60943ea1:32913 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:1;2eda60943ea1:43797 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:2;2eda60943ea1:34841 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:1;2eda60943ea1:43797 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:49:26,527 DEBUG [RS:0;2eda60943ea1:32913 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:49:26,527 DEBUG [RS:2;2eda60943ea1:34841 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:2;2eda60943ea1:34841 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:1;2eda60943ea1:43797 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:0;2eda60943ea1:32913 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:2;2eda60943ea1:34841 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:1;2eda60943ea1:43797 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:0;2eda60943ea1:32913 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,527 DEBUG [RS:2;2eda60943ea1:34841 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,528 DEBUG [RS:1;2eda60943ea1:43797 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,528 DEBUG [RS:2;2eda60943ea1:34841 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,528 DEBUG [RS:0;2eda60943ea1:32913 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,528 DEBUG [RS:2;2eda60943ea1:34841 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:49:26,528 DEBUG [RS:1;2eda60943ea1:43797 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,528 DEBUG [RS:0;2eda60943ea1:32913 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,528 DEBUG [RS:2;2eda60943ea1:34841 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:49:26,528 DEBUG [RS:1;2eda60943ea1:43797 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,528 DEBUG [RS:0;2eda60943ea1:32913 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,528 DEBUG [RS:1;2eda60943ea1:43797 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,528 DEBUG [RS:0;2eda60943ea1:32913 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:49:26,528 DEBUG [RS:1;2eda60943ea1:43797 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:49:26,528 DEBUG [RS:0;2eda60943ea1:32913 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:49:26,528 DEBUG [RS:1;2eda60943ea1:43797 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:49:26,528 DEBUG [RS:0;2eda60943ea1:32913 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:49:26,530 INFO [RS:0;2eda60943ea1:32913 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,530 INFO [RS:0;2eda60943ea1:32913 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,530 INFO [RS:0;2eda60943ea1:32913 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,530 INFO [RS:2;2eda60943ea1:34841 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,531 INFO [RS:0;2eda60943ea1:32913 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,531 INFO [RS:2;2eda60943ea1:34841 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,531 INFO [RS:0;2eda60943ea1:32913 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,531 INFO [RS:2;2eda60943ea1:34841 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,531 INFO [RS:0;2eda60943ea1:32913 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,32913,1732387765000-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:49:26,531 INFO [RS:2;2eda60943ea1:34841 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,531 INFO [RS:2;2eda60943ea1:34841 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,531 INFO [RS:2;2eda60943ea1:34841 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,34841,1732387765147-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:49:26,532 INFO [RS:1;2eda60943ea1:43797 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,532 INFO [RS:1;2eda60943ea1:43797 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,532 INFO [RS:1;2eda60943ea1:43797 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,532 INFO [RS:1;2eda60943ea1:43797 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,532 INFO [RS:1;2eda60943ea1:43797 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,532 INFO [RS:1;2eda60943ea1:43797 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,43797,1732387765099-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:49:26,551 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T18:49:26,553 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T18:49:26,553 INFO [RS:0;2eda60943ea1:32913 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,32913,1732387765000-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,553 INFO [RS:2;2eda60943ea1:34841 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,34841,1732387765147-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,553 INFO [RS:0;2eda60943ea1:32913 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,553 INFO [RS:2;2eda60943ea1:34841 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,553 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.Replication(171): 2eda60943ea1,32913,1732387765000 started 2024-11-23T18:49:26,553 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.Replication(171): 2eda60943ea1,34841,1732387765147 started 2024-11-23T18:49:26,558 WARN [2eda60943ea1:37117 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T18:49:26,560 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T18:49:26,561 INFO [RS:1;2eda60943ea1:43797 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,43797,1732387765099-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,561 INFO [RS:1;2eda60943ea1:43797 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,561 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.Replication(171): 2eda60943ea1,43797,1732387765099 started 2024-11-23T18:49:26,572 INFO [RS:2;2eda60943ea1:34841 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,572 INFO [RS:0;2eda60943ea1:32913 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,573 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.HRegionServer(1482): Serving as 2eda60943ea1,32913,1732387765000, RpcServer on 2eda60943ea1/172.17.0.2:32913, sessionid=0x101690770ba0001 2024-11-23T18:49:26,573 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.HRegionServer(1482): Serving as 2eda60943ea1,34841,1732387765147, RpcServer on 2eda60943ea1/172.17.0.2:34841, sessionid=0x101690770ba0003 2024-11-23T18:49:26,574 DEBUG [RS:0;2eda60943ea1:32913 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T18:49:26,574 DEBUG [RS:2;2eda60943ea1:34841 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T18:49:26,574 DEBUG [RS:0;2eda60943ea1:32913 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2eda60943ea1,32913,1732387765000 2024-11-23T18:49:26,574 DEBUG [RS:2;2eda60943ea1:34841 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2eda60943ea1,34841,1732387765147 2024-11-23T18:49:26,574 DEBUG [RS:0;2eda60943ea1:32913 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,32913,1732387765000' 2024-11-23T18:49:26,574 DEBUG [RS:2;2eda60943ea1:34841 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,34841,1732387765147' 2024-11-23T18:49:26,574 DEBUG [RS:0;2eda60943ea1:32913 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T18:49:26,574 DEBUG [RS:2;2eda60943ea1:34841 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T18:49:26,575 DEBUG [RS:0;2eda60943ea1:32913 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T18:49:26,575 DEBUG [RS:2;2eda60943ea1:34841 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T18:49:26,575 DEBUG [RS:0;2eda60943ea1:32913 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T18:49:26,575 DEBUG [RS:2;2eda60943ea1:34841 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T18:49:26,575 DEBUG [RS:0;2eda60943ea1:32913 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T18:49:26,575 DEBUG [RS:2;2eda60943ea1:34841 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T18:49:26,576 DEBUG [RS:2;2eda60943ea1:34841 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2eda60943ea1,34841,1732387765147 2024-11-23T18:49:26,576 DEBUG [RS:0;2eda60943ea1:32913 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2eda60943ea1,32913,1732387765000 2024-11-23T18:49:26,576 DEBUG [RS:2;2eda60943ea1:34841 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,34841,1732387765147' 2024-11-23T18:49:26,576 DEBUG [RS:0;2eda60943ea1:32913 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,32913,1732387765000' 2024-11-23T18:49:26,576 DEBUG [RS:2;2eda60943ea1:34841 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T18:49:26,576 DEBUG [RS:0;2eda60943ea1:32913 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T18:49:26,576 DEBUG [RS:2;2eda60943ea1:34841 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T18:49:26,576 DEBUG [RS:0;2eda60943ea1:32913 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T18:49:26,577 DEBUG [RS:2;2eda60943ea1:34841 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T18:49:26,577 DEBUG [RS:0;2eda60943ea1:32913 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T18:49:26,577 INFO [RS:2;2eda60943ea1:34841 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T18:49:26,577 INFO [RS:0;2eda60943ea1:32913 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T18:49:26,577 INFO [RS:2;2eda60943ea1:34841 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T18:49:26,577 INFO [RS:0;2eda60943ea1:32913 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T18:49:26,584 INFO [RS:1;2eda60943ea1:43797 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:26,584 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer(1482): Serving as 2eda60943ea1,43797,1732387765099, RpcServer on 2eda60943ea1/172.17.0.2:43797, sessionid=0x101690770ba0002 2024-11-23T18:49:26,585 DEBUG [RS:1;2eda60943ea1:43797 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T18:49:26,585 DEBUG [RS:1;2eda60943ea1:43797 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2eda60943ea1,43797,1732387765099 2024-11-23T18:49:26,585 DEBUG [RS:1;2eda60943ea1:43797 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,43797,1732387765099' 2024-11-23T18:49:26,585 DEBUG [RS:1;2eda60943ea1:43797 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T18:49:26,586 DEBUG [RS:1;2eda60943ea1:43797 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T18:49:26,586 DEBUG [RS:1;2eda60943ea1:43797 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T18:49:26,586 DEBUG [RS:1;2eda60943ea1:43797 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T18:49:26,586 DEBUG [RS:1;2eda60943ea1:43797 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2eda60943ea1,43797,1732387765099 2024-11-23T18:49:26,587 DEBUG [RS:1;2eda60943ea1:43797 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,43797,1732387765099' 2024-11-23T18:49:26,587 DEBUG [RS:1;2eda60943ea1:43797 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T18:49:26,587 DEBUG [RS:1;2eda60943ea1:43797 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T18:49:26,588 DEBUG [RS:1;2eda60943ea1:43797 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T18:49:26,588 INFO [RS:1;2eda60943ea1:43797 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T18:49:26,588 INFO [RS:1;2eda60943ea1:43797 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T18:49:26,683 INFO [RS:0;2eda60943ea1:32913 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:49:26,684 INFO [RS:2;2eda60943ea1:34841 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:49:26,687 INFO [RS:0;2eda60943ea1:32913 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C32913%2C1732387765000, suffix=, logDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000, archiveDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/oldWALs, maxLogs=32 2024-11-23T18:49:26,687 INFO [RS:2;2eda60943ea1:34841 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C34841%2C1732387765147, suffix=, logDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,34841,1732387765147, archiveDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/oldWALs, maxLogs=32 2024-11-23T18:49:26,689 INFO [RS:1;2eda60943ea1:43797 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:49:26,691 INFO [RS:1;2eda60943ea1:43797 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C43797%2C1732387765099, suffix=, logDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,43797,1732387765099, archiveDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/oldWALs, maxLogs=32 2024-11-23T18:49:26,705 DEBUG [RS:0;2eda60943ea1:32913 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000/2eda60943ea1%2C32913%2C1732387765000.1732387766690, exclude list is [], retry=0 2024-11-23T18:49:26,706 DEBUG [RS:2;2eda60943ea1:34841 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,34841,1732387765147/2eda60943ea1%2C34841%2C1732387765147.1732387766690, exclude list is [], retry=0 2024-11-23T18:49:26,706 DEBUG [RS:1;2eda60943ea1:43797 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,43797,1732387765099/2eda60943ea1%2C43797%2C1732387765099.1732387766693, exclude list is [], retry=0 2024-11-23T18:49:26,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:26,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:26,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:26,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:26,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:26,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:26,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:26,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:26,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:26,717 INFO [RS:2;2eda60943ea1:34841 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,34841,1732387765147/2eda60943ea1%2C34841%2C1732387765147.1732387766690 2024-11-23T18:49:26,717 INFO [RS:0;2eda60943ea1:32913 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000/2eda60943ea1%2C32913%2C1732387765000.1732387766690 2024-11-23T18:49:26,717 DEBUG [RS:2;2eda60943ea1:34841 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:37451:37451)] 2024-11-23T18:49:26,717 DEBUG [RS:0;2eda60943ea1:32913 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:43381:43381)] 2024-11-23T18:49:26,717 INFO [RS:1;2eda60943ea1:43797 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,43797,1732387765099/2eda60943ea1%2C43797%2C1732387765099.1732387766693 2024-11-23T18:49:26,718 DEBUG [RS:1;2eda60943ea1:43797 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:37451:37451)] 2024-11-23T18:49:26,810 DEBUG [2eda60943ea1:37117 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-23T18:49:26,818 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(204): Hosts are {2eda60943ea1=0} racks are {/default-rack=0} 2024-11-23T18:49:26,824 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T18:49:26,824 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T18:49:26,824 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-23T18:49:26,824 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T18:49:26,824 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T18:49:26,824 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-23T18:49:26,824 INFO [2eda60943ea1:37117 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T18:49:26,824 INFO [2eda60943ea1:37117 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T18:49:26,824 INFO [2eda60943ea1:37117 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-23T18:49:26,824 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T18:49:26,830 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2eda60943ea1,32913,1732387765000 2024-11-23T18:49:26,835 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,32913,1732387765000, state=OPENING 2024-11-23T18:49:26,899 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T18:49:26,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:26,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:26,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:26,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32913-0x101690770ba0001, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:49:26,915 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:49:26,915 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:49:26,915 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:49:26,915 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:49:26,918 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:49:26,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,32913,1732387765000}] 2024-11-23T18:49:27,102 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T18:49:27,105 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58825, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T18:49:27,117 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T18:49:27,117 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:27,118 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T18:49:27,121 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C32913%2C1732387765000.meta, suffix=.meta, logDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000, archiveDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/oldWALs, maxLogs=32 2024-11-23T18:49:27,137 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000/2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta, exclude list is [], retry=0 2024-11-23T18:49:27,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:27,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:27,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:27,144 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000/2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta 2024-11-23T18:49:27,145 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:43381:43381)] 2024-11-23T18:49:27,145 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:27,147 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T18:49:27,149 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T18:49:27,154 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T18:49:27,159 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T18:49:27,160 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:27,160 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T18:49:27,160 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T18:49:27,164 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T18:49:27,165 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T18:49:27,165 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:27,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:49:27,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T18:49:27,167 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T18:49:27,168 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:27,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:49:27,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T18:49:27,170 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T18:49:27,170 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:27,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:49:27,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T18:49:27,172 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T18:49:27,172 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:27,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:49:27,173 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T18:49:27,175 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740 2024-11-23T18:49:27,178 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740 2024-11-23T18:49:27,180 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T18:49:27,181 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T18:49:27,182 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T18:49:27,184 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T18:49:27,186 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66761142, jitterRate=-0.005181461572647095}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T18:49:27,187 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T18:49:27,188 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732387767161Writing region info on filesystem at 1732387767161Initializing all the Stores at 1732387767163 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387767163Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387767163Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387767163Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387767163Cleaning up temporary data from old regions at 1732387767181 (+18 ms)Running coprocessor post-open hooks at 1732387767187 (+6 ms)Region opened successfully at 1732387767188 (+1 ms) 2024-11-23T18:49:27,196 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732387767094 2024-11-23T18:49:27,207 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T18:49:27,207 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T18:49:27,209 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2eda60943ea1,32913,1732387765000 2024-11-23T18:49:27,211 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,32913,1732387765000, state=OPEN 2024-11-23T18:49:27,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:49:27,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:49:27,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:49:27,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32913-0x101690770ba0001, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:49:27,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:49:27,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:49:27,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:49:27,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:49:27,225 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2eda60943ea1,32913,1732387765000 2024-11-23T18:49:27,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T18:49:27,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,32913,1732387765000 in 305 msec 2024-11-23T18:49:27,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T18:49:27,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 838 msec 2024-11-23T18:49:27,243 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:49:27,243 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T18:49:27,261 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:49:27,262 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1] 2024-11-23T18:49:27,281 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:49:27,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53059, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:49:27,303 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1270 sec 2024-11-23T18:49:27,303 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732387767303, completionTime=-1 2024-11-23T18:49:27,306 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-23T18:49:27,306 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T18:49:27,349 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-23T18:49:27,349 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732387827349 2024-11-23T18:49:27,349 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732387887349 2024-11-23T18:49:27,349 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 42 msec 2024-11-23T18:49:27,351 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-23T18:49:27,359 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37117,1732387764258-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:27,360 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37117,1732387764258-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:27,360 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37117,1732387764258-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:27,362 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2eda60943ea1:37117, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:27,362 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:27,363 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:27,373 DEBUG [master/2eda60943ea1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T18:49:27,397 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.145sec 2024-11-23T18:49:27,398 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T18:49:27,399 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T18:49:27,400 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T18:49:27,401 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T18:49:27,401 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T18:49:27,401 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37117,1732387764258-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:49:27,402 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37117,1732387764258-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T18:49:27,406 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T18:49:27,407 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T18:49:27,407 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37117,1732387764258-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:27,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ddf6023, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:49:27,488 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2eda60943ea1,37117,-1 for getting cluster id 2024-11-23T18:49:27,491 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T18:49:27,501 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c7282851-006a-445f-a51c-1e6fe6ae5ccd' 2024-11-23T18:49:27,503 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T18:49:27,503 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c7282851-006a-445f-a51c-1e6fe6ae5ccd" 2024-11-23T18:49:27,504 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1933dd94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:49:27,504 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2eda60943ea1,37117,-1] 2024-11-23T18:49:27,507 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T18:49:27,509 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:49:27,510 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T18:49:27,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65c83e31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:49:27,513 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:49:27,519 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1] 2024-11-23T18:49:27,520 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:49:27,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:49:27,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2eda60943ea1,37117,1732387764258 2024-11-23T18:49:27,540 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:43835/hbase 2024-11-23T18:49:27,552 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=364, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=270, ProcessCount=11, AvailableMemoryMB=9087 2024-11-23T18:49:27,572 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:27,575 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:27,576 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:49:27,581 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-02594430, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/hregion-02594430, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:27,596 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-02594430/hregion-02594430.1732387767583, exclude list is [], retry=0 2024-11-23T18:49:27,600 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:27,601 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:27,601 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:27,604 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-02594430/hregion-02594430.1732387767583 2024-11-23T18:49:27,606 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:43381:43381)] 2024-11-23T18:49:27,606 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5f7a9c69b5ea59aff7c1c671d1d4fcf0, NAME => 'testReplayEditsWrittenIntoWAL,,1732387767573.5f7a9c69b5ea59aff7c1c671d1d4fcf0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43835/hbase 2024-11-23T18:49:27,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741838_1014 (size=64) 2024-11-23T18:49:27,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741838_1014 (size=64) 2024-11-23T18:49:27,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741838_1014 (size=64) 2024-11-23T18:49:27,621 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732387767573.5f7a9c69b5ea59aff7c1c671d1d4fcf0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:27,623 INFO [StoreOpener-5f7a9c69b5ea59aff7c1c671d1d4fcf0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5f7a9c69b5ea59aff7c1c671d1d4fcf0 2024-11-23T18:49:27,625 INFO [StoreOpener-5f7a9c69b5ea59aff7c1c671d1d4fcf0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f7a9c69b5ea59aff7c1c671d1d4fcf0 columnFamilyName a 2024-11-23T18:49:27,625 DEBUG [StoreOpener-5f7a9c69b5ea59aff7c1c671d1d4fcf0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:27,626 INFO [StoreOpener-5f7a9c69b5ea59aff7c1c671d1d4fcf0-1 {}] regionserver.HStore(327): Store=5f7a9c69b5ea59aff7c1c671d1d4fcf0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:27,627 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5f7a9c69b5ea59aff7c1c671d1d4fcf0 2024-11-23T18:49:27,628 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0 2024-11-23T18:49:27,628 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0 2024-11-23T18:49:27,629 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5f7a9c69b5ea59aff7c1c671d1d4fcf0 2024-11-23T18:49:27,629 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5f7a9c69b5ea59aff7c1c671d1d4fcf0 2024-11-23T18:49:27,632 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5f7a9c69b5ea59aff7c1c671d1d4fcf0 2024-11-23T18:49:27,635 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:49:27,636 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5f7a9c69b5ea59aff7c1c671d1d4fcf0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62874500, jitterRate=-0.06309694051742554}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:49:27,637 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5f7a9c69b5ea59aff7c1c671d1d4fcf0: Writing region info on filesystem at 1732387767621Initializing all the Stores at 1732387767622 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387767622Cleaning up temporary data from old regions at 1732387767629 (+7 ms)Region opened successfully at 1732387767637 (+8 ms) 2024-11-23T18:49:27,637 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5f7a9c69b5ea59aff7c1c671d1d4fcf0, disabling compactions & flushes 2024-11-23T18:49:27,637 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732387767573.5f7a9c69b5ea59aff7c1c671d1d4fcf0. 2024-11-23T18:49:27,637 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732387767573.5f7a9c69b5ea59aff7c1c671d1d4fcf0. 2024-11-23T18:49:27,637 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732387767573.5f7a9c69b5ea59aff7c1c671d1d4fcf0. after waiting 0 ms 2024-11-23T18:49:27,637 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732387767573.5f7a9c69b5ea59aff7c1c671d1d4fcf0. 2024-11-23T18:49:27,638 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732387767573.5f7a9c69b5ea59aff7c1c671d1d4fcf0. 2024-11-23T18:49:27,638 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5f7a9c69b5ea59aff7c1c671d1d4fcf0: Waiting for close lock at 1732387767637Disabling compacts and flushes for region at 1732387767637Disabling writes for close at 1732387767637Writing region close event to WAL at 1732387767638 (+1 ms)Closed at 1732387767638 2024-11-23T18:49:27,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741837_1013 (size=95) 2024-11-23T18:49:27,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741837_1013 (size=95) 2024-11-23T18:49:27,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741837_1013 (size=95) 2024-11-23T18:49:27,649 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:49:27,649 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-02594430:(num 1732387767583) 2024-11-23T18:49:27,651 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-23T18:49:27,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741839_1015 (size=320) 2024-11-23T18:49:27,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741839_1015 (size=320) 2024-11-23T18:49:27,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741839_1015 (size=320) 2024-11-23T18:49:27,672 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-23T18:49:27,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741840_1016 (size=253) 2024-11-23T18:49:27,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741840_1016 (size=253) 2024-11-23T18:49:27,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741840_1016 (size=253) 2024-11-23T18:49:27,707 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-1, size=320 (320bytes) 2024-11-23T18:49:27,708 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-23T18:49:27,708 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-23T18:49:27,708 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-1 2024-11-23T18:49:27,714 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-1 after 4ms 2024-11-23T18:49:27,719 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:27,720 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-1 took 14ms 2024-11-23T18:49:27,730 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-1 so closing down 2024-11-23T18:49:27,730 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:49:27,733 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-23T18:49:27,735 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000001-wal-1.temp 2024-11-23T18:49:27,737 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:49:27,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741841_1017 (size=320) 2024-11-23T18:49:27,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741841_1017 (size=320) 2024-11-23T18:49:27,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741841_1017 (size=320) 2024-11-23T18:49:27,749 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-23T18:49:27,751 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000002 2024-11-23T18:49:27,755 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-23T18:49:27,755 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-1, journal: Splitting hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-1, size=320 (320bytes) at 1732387767707Finishing writing output for hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-1 so closing down at 1732387767730 (+23 ms)Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000001-wal-1.temp at 1732387767735 (+5 ms)3 split writer threads finished at 1732387767737 (+2 ms)Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732387767749 (+12 ms)Rename recovered edits hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000002 at 1732387767751 (+2 ms)Processed 2 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1732387767755 (+4 ms) 2024-11-23T18:49:27,770 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-2, size=253 (253bytes) 2024-11-23T18:49:27,770 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-2 2024-11-23T18:49:27,771 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-2 after 1ms 2024-11-23T18:49:27,775 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:27,776 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-2 took 6ms 2024-11-23T18:49:27,778 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-2 so closing down 2024-11-23T18:49:27,778 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:49:27,781 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-23T18:49:27,782 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000002-wal-2.temp 2024-11-23T18:49:27,783 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:49:27,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741842_1018 (size=253) 2024-11-23T18:49:27,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741842_1018 (size=253) 2024-11-23T18:49:27,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741842_1018 (size=253) 2024-11-23T18:49:27,793 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-23T18:49:27,797 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:27,800 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000002-wal-2.temp, length=253 2024-11-23T18:49:27,802 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-23T18:49:27,802 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-2, journal: Splitting hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-2, size=253 (253bytes) at 1732387767770Finishing writing output for hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-2 so closing down at 1732387767778 (+8 ms)Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000002-wal-2.temp at 1732387767782 (+4 ms)3 split writer threads finished at 1732387767783 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732387767793 (+10 ms)Processed 1 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1732387767802 (+9 ms) 2024-11-23T18:49:27,802 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:49:27,805 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:27,818 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal.1732387767805, exclude list is [], retry=0 2024-11-23T18:49:27,822 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:27,823 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:27,823 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:27,826 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal.1732387767805 2024-11-23T18:49:27,826 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:43381:43381)] 2024-11-23T18:49:27,826 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f7a9c69b5ea59aff7c1c671d1d4fcf0, NAME => 'testReplayEditsWrittenIntoWAL,,1732387767573.5f7a9c69b5ea59aff7c1c671d1d4fcf0.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:27,826 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732387767573.5f7a9c69b5ea59aff7c1c671d1d4fcf0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:27,827 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5f7a9c69b5ea59aff7c1c671d1d4fcf0 2024-11-23T18:49:27,827 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5f7a9c69b5ea59aff7c1c671d1d4fcf0 2024-11-23T18:49:27,829 INFO [StoreOpener-5f7a9c69b5ea59aff7c1c671d1d4fcf0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5f7a9c69b5ea59aff7c1c671d1d4fcf0 2024-11-23T18:49:27,830 INFO [StoreOpener-5f7a9c69b5ea59aff7c1c671d1d4fcf0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f7a9c69b5ea59aff7c1c671d1d4fcf0 columnFamilyName a 2024-11-23T18:49:27,830 DEBUG [StoreOpener-5f7a9c69b5ea59aff7c1c671d1d4fcf0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:27,832 INFO [StoreOpener-5f7a9c69b5ea59aff7c1c671d1d4fcf0-1 {}] regionserver.HStore(327): Store=5f7a9c69b5ea59aff7c1c671d1d4fcf0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:27,832 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5f7a9c69b5ea59aff7c1c671d1d4fcf0 2024-11-23T18:49:27,833 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0 2024-11-23T18:49:27,837 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0 2024-11-23T18:49:27,838 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000002 2024-11-23T18:49:27,842 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:27,847 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000002 2024-11-23T18:49:27,851 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5f7a9c69b5ea59aff7c1c671d1d4fcf0 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-23T18:49:27,899 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/.tmp/a/781d90e3e64f4049b3e52f747a5e292d is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732387767649/Put/seqid=0 2024-11-23T18:49:27,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741844_1020 (size=5170) 2024-11-23T18:49:27,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741844_1020 (size=5170) 2024-11-23T18:49:27,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741844_1020 (size=5170) 2024-11-23T18:49:27,912 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/.tmp/a/781d90e3e64f4049b3e52f747a5e292d 2024-11-23T18:49:27,954 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/.tmp/a/781d90e3e64f4049b3e52f747a5e292d as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/a/781d90e3e64f4049b3e52f747a5e292d 2024-11-23T18:49:27,963 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/a/781d90e3e64f4049b3e52f747a5e292d, entries=2, sequenceid=2, filesize=5.0 K 2024-11-23T18:49:27,968 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 5f7a9c69b5ea59aff7c1c671d1d4fcf0 in 116ms, sequenceid=2, compaction requested=false; wal=null 2024-11-23T18:49:27,970 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/0000000000000000002 2024-11-23T18:49:27,970 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5f7a9c69b5ea59aff7c1c671d1d4fcf0 2024-11-23T18:49:27,970 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5f7a9c69b5ea59aff7c1c671d1d4fcf0 2024-11-23T18:49:27,973 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5f7a9c69b5ea59aff7c1c671d1d4fcf0 2024-11-23T18:49:27,977 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/5f7a9c69b5ea59aff7c1c671d1d4fcf0/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-23T18:49:27,978 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5f7a9c69b5ea59aff7c1c671d1d4fcf0; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63587520, jitterRate=-0.05247211456298828}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:49:27,979 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5f7a9c69b5ea59aff7c1c671d1d4fcf0: Writing region info on filesystem at 1732387767827Initializing all the Stores at 1732387767829 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387767829Obtaining lock to block concurrent updates at 1732387767851 (+22 ms)Preparing flush snapshotting stores in 5f7a9c69b5ea59aff7c1c671d1d4fcf0 at 1732387767851Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732387767573.5f7a9c69b5ea59aff7c1c671d1d4fcf0., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732387767854 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732387767573.5f7a9c69b5ea59aff7c1c671d1d4fcf0. at 1732387767854Flushing 5f7a9c69b5ea59aff7c1c671d1d4fcf0/a: creating writer at 1732387767855 (+1 ms)Flushing 5f7a9c69b5ea59aff7c1c671d1d4fcf0/a: appending metadata at 1732387767888 (+33 ms)Flushing 5f7a9c69b5ea59aff7c1c671d1d4fcf0/a: closing flushed file at 1732387767890 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3dc95ff4: reopening flushed file at 1732387767952 (+62 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 5f7a9c69b5ea59aff7c1c671d1d4fcf0 in 116ms, sequenceid=2, compaction requested=false; wal=null at 1732387767968 (+16 ms)Cleaning up temporary data from old regions at 1732387767970 (+2 ms)Region opened successfully at 1732387767979 (+9 ms) 2024-11-23T18:49:28,005 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=375 (was 364) Potentially hanging thread: PacketResponder: BP-995492633-172.17.0.2-1732387759654:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-995492633-172.17.0.2-1732387759654:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:53184 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:42432 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:53264 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:43835/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:40774 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:40596 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-995492633-172.17.0.2-1732387759654:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:42310 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=272 (was 270) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9065 (was 9087) 2024-11-23T18:49:28,016 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=375, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=272, ProcessCount=11, AvailableMemoryMB=9064 2024-11-23T18:49:28,033 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:28,035 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:28,036 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:49:28,039 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-20830704, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/hregion-20830704, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:28,051 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-20830704/hregion-20830704.1732387768039, exclude list is [], retry=0 2024-11-23T18:49:28,055 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:28,055 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:28,055 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:28,058 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-20830704/hregion-20830704.1732387768039 2024-11-23T18:49:28,062 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:39619:39619)] 2024-11-23T18:49:28,062 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b26e195fb6cdfcbf8e44365006408486, NAME => 'testReplayEditsWrittenIntoWAL,,1732387768033.b26e195fb6cdfcbf8e44365006408486.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43835/hbase 2024-11-23T18:49:28,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741846_1022 (size=64) 2024-11-23T18:49:28,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741846_1022 (size=64) 2024-11-23T18:49:28,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741846_1022 (size=64) 2024-11-23T18:49:28,074 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732387768033.b26e195fb6cdfcbf8e44365006408486.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:28,076 INFO [StoreOpener-b26e195fb6cdfcbf8e44365006408486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b26e195fb6cdfcbf8e44365006408486 2024-11-23T18:49:28,078 INFO [StoreOpener-b26e195fb6cdfcbf8e44365006408486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b26e195fb6cdfcbf8e44365006408486 columnFamilyName a 2024-11-23T18:49:28,078 DEBUG [StoreOpener-b26e195fb6cdfcbf8e44365006408486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:28,079 INFO [StoreOpener-b26e195fb6cdfcbf8e44365006408486-1 {}] regionserver.HStore(327): Store=b26e195fb6cdfcbf8e44365006408486/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:28,079 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b26e195fb6cdfcbf8e44365006408486 2024-11-23T18:49:28,081 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486 2024-11-23T18:49:28,081 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486 2024-11-23T18:49:28,082 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b26e195fb6cdfcbf8e44365006408486 2024-11-23T18:49:28,082 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b26e195fb6cdfcbf8e44365006408486 2024-11-23T18:49:28,084 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b26e195fb6cdfcbf8e44365006408486 2024-11-23T18:49:28,086 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:49:28,087 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b26e195fb6cdfcbf8e44365006408486; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71520083, jitterRate=0.06573228538036346}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:49:28,087 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b26e195fb6cdfcbf8e44365006408486: Writing region info on filesystem at 1732387768074Initializing all the Stores at 1732387768075 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387768075Cleaning up temporary data from old regions at 1732387768082 (+7 ms)Region opened successfully at 1732387768087 (+5 ms) 2024-11-23T18:49:28,087 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b26e195fb6cdfcbf8e44365006408486, disabling compactions & flushes 2024-11-23T18:49:28,087 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732387768033.b26e195fb6cdfcbf8e44365006408486. 2024-11-23T18:49:28,087 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732387768033.b26e195fb6cdfcbf8e44365006408486. 2024-11-23T18:49:28,087 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732387768033.b26e195fb6cdfcbf8e44365006408486. after waiting 0 ms 2024-11-23T18:49:28,087 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732387768033.b26e195fb6cdfcbf8e44365006408486. 2024-11-23T18:49:28,088 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732387768033.b26e195fb6cdfcbf8e44365006408486. 2024-11-23T18:49:28,088 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b26e195fb6cdfcbf8e44365006408486: Waiting for close lock at 1732387768087Disabling compacts and flushes for region at 1732387768087Disabling writes for close at 1732387768087Writing region close event to WAL at 1732387768088 (+1 ms)Closed at 1732387768088 2024-11-23T18:49:28,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741845_1021 (size=95) 2024-11-23T18:49:28,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741845_1021 (size=95) 2024-11-23T18:49:28,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741845_1021 (size=95) 2024-11-23T18:49:28,095 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:49:28,095 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-20830704:(num 1732387768039) 2024-11-23T18:49:28,096 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-23T18:49:28,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741847_1023 (size=320) 2024-11-23T18:49:28,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741847_1023 (size=320) 2024-11-23T18:49:28,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741847_1023 (size=320) 2024-11-23T18:49:28,108 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-23T18:49:28,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741848_1024 (size=253) 2024-11-23T18:49:28,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741848_1024 (size=253) 2024-11-23T18:49:28,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741848_1024 (size=253) 2024-11-23T18:49:28,136 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-2, size=253 (253bytes) 2024-11-23T18:49:28,136 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-2 2024-11-23T18:49:28,137 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-2 after 1ms 2024-11-23T18:49:28,140 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:28,141 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-2 took 5ms 2024-11-23T18:49:28,143 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-2 so closing down 2024-11-23T18:49:28,143 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:49:28,145 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-23T18:49:28,146 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000002-wal-2.temp 2024-11-23T18:49:28,147 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:49:28,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741849_1025 (size=253) 2024-11-23T18:49:28,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741849_1025 (size=253) 2024-11-23T18:49:28,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741849_1025 (size=253) 2024-11-23T18:49:28,154 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-23T18:49:28,157 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000002 2024-11-23T18:49:28,157 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-23T18:49:28,157 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-2, journal: Splitting hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-2, size=253 (253bytes) at 1732387768136Finishing writing output for hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-2 so closing down at 1732387768143 (+7 ms)Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000002-wal-2.temp at 1732387768146 (+3 ms)3 split writer threads finished at 1732387768147 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732387768154 (+7 ms)Rename recovered edits hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000002 at 1732387768157 (+3 ms)Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1732387768157 2024-11-23T18:49:28,172 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-1, size=320 (320bytes) 2024-11-23T18:49:28,173 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-1 2024-11-23T18:49:28,173 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-1 after 0ms 2024-11-23T18:49:28,177 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:28,177 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-1 took 5ms 2024-11-23T18:49:28,180 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-1 so closing down 2024-11-23T18:49:28,180 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:49:28,182 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-23T18:49:28,183 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000001-wal-1.temp 2024-11-23T18:49:28,184 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:49:28,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741850_1026 (size=320) 2024-11-23T18:49:28,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741850_1026 (size=320) 2024-11-23T18:49:28,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741850_1026 (size=320) 2024-11-23T18:49:28,191 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-23T18:49:28,195 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:28,197 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000002, length=253 2024-11-23T18:49:28,199 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000002 2024-11-23T18:49:28,200 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-23T18:49:28,200 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-1, journal: Splitting hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-1, size=320 (320bytes) at 1732387768173Finishing writing output for hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-1 so closing down at 1732387768180 (+7 ms)Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000001-wal-1.temp at 1732387768183 (+3 ms)3 split writer threads finished at 1732387768184 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732387768191 (+7 ms)Rename recovered edits hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000002 at 1732387768199 (+8 ms)Processed 2 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1732387768200 (+1 ms) 2024-11-23T18:49:28,200 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:49:28,202 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:28,217 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal.1732387768203, exclude list is [], retry=0 2024-11-23T18:49:28,221 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:28,221 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:28,221 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:28,224 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal.1732387768203 2024-11-23T18:49:28,224 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:39619:39619)] 2024-11-23T18:49:28,224 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b26e195fb6cdfcbf8e44365006408486, NAME => 'testReplayEditsWrittenIntoWAL,,1732387768033.b26e195fb6cdfcbf8e44365006408486.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:28,225 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732387768033.b26e195fb6cdfcbf8e44365006408486.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:28,225 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b26e195fb6cdfcbf8e44365006408486 2024-11-23T18:49:28,225 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b26e195fb6cdfcbf8e44365006408486 2024-11-23T18:49:28,227 INFO [StoreOpener-b26e195fb6cdfcbf8e44365006408486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b26e195fb6cdfcbf8e44365006408486 2024-11-23T18:49:28,230 INFO [StoreOpener-b26e195fb6cdfcbf8e44365006408486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b26e195fb6cdfcbf8e44365006408486 columnFamilyName a 2024-11-23T18:49:28,230 DEBUG [StoreOpener-b26e195fb6cdfcbf8e44365006408486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:28,231 INFO [StoreOpener-b26e195fb6cdfcbf8e44365006408486-1 {}] regionserver.HStore(327): Store=b26e195fb6cdfcbf8e44365006408486/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:28,231 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b26e195fb6cdfcbf8e44365006408486 2024-11-23T18:49:28,232 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486 2024-11-23T18:49:28,235 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486 2024-11-23T18:49:28,236 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000002 2024-11-23T18:49:28,239 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:28,241 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000002 2024-11-23T18:49:28,241 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b26e195fb6cdfcbf8e44365006408486 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-23T18:49:28,258 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/.tmp/a/7966cbe044cd48f3a08860a4328d8621 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732387768095/Put/seqid=0 2024-11-23T18:49:28,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741852_1028 (size=5170) 2024-11-23T18:49:28,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741852_1028 (size=5170) 2024-11-23T18:49:28,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741852_1028 (size=5170) 2024-11-23T18:49:28,268 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/.tmp/a/7966cbe044cd48f3a08860a4328d8621 2024-11-23T18:49:28,279 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/.tmp/a/7966cbe044cd48f3a08860a4328d8621 as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/a/7966cbe044cd48f3a08860a4328d8621 2024-11-23T18:49:28,288 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/a/7966cbe044cd48f3a08860a4328d8621, entries=2, sequenceid=2, filesize=5.0 K 2024-11-23T18:49:28,288 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for b26e195fb6cdfcbf8e44365006408486 in 47ms, sequenceid=2, compaction requested=false; wal=null 2024-11-23T18:49:28,289 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/0000000000000000002 2024-11-23T18:49:28,290 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b26e195fb6cdfcbf8e44365006408486 2024-11-23T18:49:28,290 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b26e195fb6cdfcbf8e44365006408486 2024-11-23T18:49:28,293 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b26e195fb6cdfcbf8e44365006408486 2024-11-23T18:49:28,295 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/b26e195fb6cdfcbf8e44365006408486/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-23T18:49:28,297 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b26e195fb6cdfcbf8e44365006408486; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66426065, jitterRate=-0.010174497961997986}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:49:28,297 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b26e195fb6cdfcbf8e44365006408486: Writing region info on filesystem at 1732387768225Initializing all the Stores at 1732387768227 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387768227Obtaining lock to block concurrent updates at 1732387768241 (+14 ms)Preparing flush snapshotting stores in b26e195fb6cdfcbf8e44365006408486 at 1732387768241Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732387768033.b26e195fb6cdfcbf8e44365006408486., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732387768241Flushing stores of testReplayEditsWrittenIntoWAL,,1732387768033.b26e195fb6cdfcbf8e44365006408486. at 1732387768241Flushing b26e195fb6cdfcbf8e44365006408486/a: creating writer at 1732387768241Flushing b26e195fb6cdfcbf8e44365006408486/a: appending metadata at 1732387768257 (+16 ms)Flushing b26e195fb6cdfcbf8e44365006408486/a: closing flushed file at 1732387768257Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4785ed3c: reopening flushed file at 1732387768277 (+20 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for b26e195fb6cdfcbf8e44365006408486 in 47ms, sequenceid=2, compaction requested=false; wal=null at 1732387768288 (+11 ms)Cleaning up temporary data from old regions at 1732387768290 (+2 ms)Region opened successfully at 1732387768297 (+7 ms) 2024-11-23T18:49:28,316 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=385 (was 375) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:42490 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-995492633-172.17.0.2-1732387759654:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:53340 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:53184 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-995492633-172.17.0.2-1732387759654:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-995492633-172.17.0.2-1732387759654:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:40838 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:40596 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:42310 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=272 (was 272), ProcessCount=11 (was 11), AvailableMemoryMB=9056 (was 9064) 2024-11-23T18:49:28,326 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=385, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=272, ProcessCount=11, AvailableMemoryMB=9054 2024-11-23T18:49:28,342 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:28,344 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:28,345 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:49:28,348 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-89547218, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/hregion-89547218, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:28,363 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-89547218/hregion-89547218.1732387768349, exclude list is [], retry=0 2024-11-23T18:49:28,366 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:28,366 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:28,366 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:28,369 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-89547218/hregion-89547218.1732387768349 2024-11-23T18:49:28,369 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:37451:37451)] 2024-11-23T18:49:28,369 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 45ce04fe4b79f00ca440f4fae21b7a85, NAME => 'testReplayEditsWrittenIntoWAL,,1732387768342.45ce04fe4b79f00ca440f4fae21b7a85.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43835/hbase 2024-11-23T18:49:28,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741854_1030 (size=64) 2024-11-23T18:49:28,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741854_1030 (size=64) 2024-11-23T18:49:28,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741854_1030 (size=64) 2024-11-23T18:49:28,380 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732387768342.45ce04fe4b79f00ca440f4fae21b7a85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:28,382 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:28,383 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45ce04fe4b79f00ca440f4fae21b7a85 columnFamilyName a 2024-11-23T18:49:28,384 DEBUG [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:28,384 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] regionserver.HStore(327): Store=45ce04fe4b79f00ca440f4fae21b7a85/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:28,384 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:28,386 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45ce04fe4b79f00ca440f4fae21b7a85 columnFamilyName b 2024-11-23T18:49:28,386 DEBUG [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:28,387 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] regionserver.HStore(327): Store=45ce04fe4b79f00ca440f4fae21b7a85/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:28,387 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:28,389 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45ce04fe4b79f00ca440f4fae21b7a85 columnFamilyName c 2024-11-23T18:49:28,389 DEBUG [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:28,390 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] regionserver.HStore(327): Store=45ce04fe4b79f00ca440f4fae21b7a85/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:28,390 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:28,391 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:28,391 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:28,393 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:28,393 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:28,393 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:49:28,395 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:28,397 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:49:28,398 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 45ce04fe4b79f00ca440f4fae21b7a85; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68836196, jitterRate=0.02573925256729126}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:49:28,398 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 45ce04fe4b79f00ca440f4fae21b7a85: Writing region info on filesystem at 1732387768380Initializing all the Stores at 1732387768381 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387768381Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387768382 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387768382Cleaning up temporary data from old regions at 1732387768393 (+11 ms)Region opened successfully at 1732387768398 (+5 ms) 2024-11-23T18:49:28,398 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 45ce04fe4b79f00ca440f4fae21b7a85, disabling compactions & flushes 2024-11-23T18:49:28,398 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732387768342.45ce04fe4b79f00ca440f4fae21b7a85. 2024-11-23T18:49:28,398 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732387768342.45ce04fe4b79f00ca440f4fae21b7a85. 2024-11-23T18:49:28,398 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732387768342.45ce04fe4b79f00ca440f4fae21b7a85. after waiting 0 ms 2024-11-23T18:49:28,398 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732387768342.45ce04fe4b79f00ca440f4fae21b7a85. 2024-11-23T18:49:28,399 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732387768342.45ce04fe4b79f00ca440f4fae21b7a85. 2024-11-23T18:49:28,399 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 45ce04fe4b79f00ca440f4fae21b7a85: Waiting for close lock at 1732387768398Disabling compacts and flushes for region at 1732387768398Disabling writes for close at 1732387768398Writing region close event to WAL at 1732387768399 (+1 ms)Closed at 1732387768399 2024-11-23T18:49:28,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741853_1029 (size=95) 2024-11-23T18:49:28,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741853_1029 (size=95) 2024-11-23T18:49:28,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741853_1029 (size=95) 2024-11-23T18:49:28,406 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:49:28,406 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-89547218:(num 1732387768349) 2024-11-23T18:49:28,406 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:49:28,408 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:28,422 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408, exclude list is [], retry=0 2024-11-23T18:49:28,425 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:28,425 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:28,426 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:28,428 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408 2024-11-23T18:49:28,429 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:39619:39619)] 2024-11-23T18:49:28,673 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408, size=0 (0bytes) 2024-11-23T18:49:28,673 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408 might be still open, length is 0 2024-11-23T18:49:28,673 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408 2024-11-23T18:49:28,674 WARN [IPC Server handler 4 on default port 43835 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-23T18:49:28,675 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408 after 1ms 2024-11-23T18:49:29,329 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:53380 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37657:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53380 dst: /127.0.0.1:37657 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37657 remote=/127.0.0.1:53380]. Total timeout mills is 60000, 59310 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:29,330 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:40858 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43523:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40858 dst: /127.0.0.1:43523 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:29,330 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:42522 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40877:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42522 dst: /127.0.0.1:40877 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:29,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741855_1032 (size=263633) 2024-11-23T18:49:29,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741855_1032 (size=263633) 2024-11-23T18:49:29,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741855_1032 (size=263633) 2024-11-23T18:49:32,676 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T18:49:32,676 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408 after 4003ms 2024-11-23T18:49:32,684 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:32,686 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408 took 4014ms 2024-11-23T18:49:32,692 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732387768408.temp 2024-11-23T18:49:32,696 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/recovered.edits/0000000000000000001-wal.1732387768408.temp 2024-11-23T18:49:32,756 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T18:49:32,846 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408; continuing. 2024-11-23T18:49:32,846 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408 so closing down 2024-11-23T18:49:32,846 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:49:32,848 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:49:32,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741856_1033 (size=263641) 2024-11-23T18:49:32,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741856_1033 (size=263641) 2024-11-23T18:49:32,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741856_1033 (size=263641) 2024-11-23T18:49:32,852 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/recovered.edits/0000000000000000001-wal.1732387768408.temp (wrote 3002 edits, skipped 0 edits in 74 ms) 2024-11-23T18:49:32,855 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/recovered.edits/0000000000000000001-wal.1732387768408.temp to hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/recovered.edits/0000000000000003002 2024-11-23T18:49:32,855 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 168 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408, size=0, length=0, corrupted=false, cancelled=false 2024-11-23T18:49:32,855 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408, journal: Splitting hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408, size=0 (0bytes) at 1732387768673Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/recovered.edits/0000000000000000001-wal.1732387768408.temp at 1732387772696 (+4023 ms)Split 1024 edits, skipped 0 edits. at 1732387772779 (+83 ms)Split 2048 edits, skipped 0 edits. at 1732387772815 (+36 ms)Finishing writing output for hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408 so closing down at 1732387772846 (+31 ms)3 split writer threads finished at 1732387772848 (+2 ms)Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/recovered.edits/0000000000000000001-wal.1732387768408.temp (wrote 3002 edits, skipped 0 edits in 74 ms) at 1732387772853 (+5 ms)Rename recovered edits hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/recovered.edits/0000000000000000001-wal.1732387768408.temp to hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/recovered.edits/0000000000000003002 at 1732387772855 (+2 ms)Processed 3002 edits across 1 Regions in 168 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408, size=0, length=0, corrupted=false, cancelled=false at 1732387772855 2024-11-23T18:49:32,857 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408 to hdfs://localhost:43835/hbase/oldWALs/wal.1732387768408 2024-11-23T18:49:32,858 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/recovered.edits/0000000000000003002 2024-11-23T18:49:32,859 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:49:32,861 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:32,875 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387772862, exclude list is [], retry=0 2024-11-23T18:49:32,878 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:32,879 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:32,879 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:32,881 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387772862 2024-11-23T18:49:32,882 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:39619:39619)] 2024-11-23T18:49:32,882 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732387768342.45ce04fe4b79f00ca440f4fae21b7a85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:32,884 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:32,886 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45ce04fe4b79f00ca440f4fae21b7a85 columnFamilyName a 2024-11-23T18:49:32,886 DEBUG [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:32,886 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] regionserver.HStore(327): Store=45ce04fe4b79f00ca440f4fae21b7a85/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:32,886 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:32,887 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45ce04fe4b79f00ca440f4fae21b7a85 columnFamilyName b 2024-11-23T18:49:32,887 DEBUG [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:32,888 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] regionserver.HStore(327): Store=45ce04fe4b79f00ca440f4fae21b7a85/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:32,888 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:32,889 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45ce04fe4b79f00ca440f4fae21b7a85 columnFamilyName c 2024-11-23T18:49:32,889 DEBUG [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:32,890 INFO [StoreOpener-45ce04fe4b79f00ca440f4fae21b7a85-1 {}] regionserver.HStore(327): Store=45ce04fe4b79f00ca440f4fae21b7a85/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:32,890 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:32,892 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:32,894 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:32,896 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/recovered.edits/0000000000000003002 2024-11-23T18:49:32,899 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:32,937 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-23T18:49:33,292 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 45ce04fe4b79f00ca440f4fae21b7a85 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-23T18:49:33,329 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/a/dafc3e55a85847f9a15ce8bb28943f96 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732387768440/Put/seqid=0 2024-11-23T18:49:33,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741858_1035 (size=50463) 2024-11-23T18:49:33,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741858_1035 (size=50463) 2024-11-23T18:49:33,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741858_1035 (size=50463) 2024-11-23T18:49:33,339 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/a/dafc3e55a85847f9a15ce8bb28943f96 2024-11-23T18:49:33,349 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/a/dafc3e55a85847f9a15ce8bb28943f96 as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/a/dafc3e55a85847f9a15ce8bb28943f96 2024-11-23T18:49:33,358 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/a/dafc3e55a85847f9a15ce8bb28943f96, entries=754, sequenceid=754, filesize=49.3 K 2024-11-23T18:49:33,358 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 45ce04fe4b79f00ca440f4fae21b7a85 in 66ms, sequenceid=754, compaction requested=false; wal=null 2024-11-23T18:49:33,391 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-23T18:49:33,392 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 45ce04fe4b79f00ca440f4fae21b7a85 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-23T18:49:33,402 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/a/bd9c7ad5205d4434b054a98da176f762 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732387768494/Put/seqid=0 2024-11-23T18:49:33,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741859_1036 (size=20072) 2024-11-23T18:49:33,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741859_1036 (size=20072) 2024-11-23T18:49:33,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741859_1036 (size=20072) 2024-11-23T18:49:33,813 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/a/bd9c7ad5205d4434b054a98da176f762 2024-11-23T18:49:33,846 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/b/1177c160370d4d159652e7be1cfe5bb0 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732387768520/Put/seqid=0 2024-11-23T18:49:33,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741860_1037 (size=35835) 2024-11-23T18:49:33,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741860_1037 (size=35835) 2024-11-23T18:49:33,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741860_1037 (size=35835) 2024-11-23T18:49:33,855 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/b/1177c160370d4d159652e7be1cfe5bb0 2024-11-23T18:49:33,863 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/a/bd9c7ad5205d4434b054a98da176f762 as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/a/bd9c7ad5205d4434b054a98da176f762 2024-11-23T18:49:33,870 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/a/bd9c7ad5205d4434b054a98da176f762, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-23T18:49:33,872 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/b/1177c160370d4d159652e7be1cfe5bb0 as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/b/1177c160370d4d159652e7be1cfe5bb0 2024-11-23T18:49:33,879 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/b/1177c160370d4d159652e7be1cfe5bb0, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-23T18:49:33,879 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 45ce04fe4b79f00ca440f4fae21b7a85 in 487ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-23T18:49:33,901 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-23T18:49:33,902 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 45ce04fe4b79f00ca440f4fae21b7a85 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-23T18:49:33,910 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/b/b51b16658bb54c82afdc8ba1bcaff6a0 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732387768549/Put/seqid=0 2024-11-23T18:49:33,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741861_1038 (size=35082) 2024-11-23T18:49:33,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741861_1038 (size=35082) 2024-11-23T18:49:33,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741861_1038 (size=35082) 2024-11-23T18:49:33,918 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/b/b51b16658bb54c82afdc8ba1bcaff6a0 2024-11-23T18:49:33,939 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/c/aa586dd25db34f6c9526f3e993cd6e68 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732387768593/Put/seqid=0 2024-11-23T18:49:33,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741862_1039 (size=20825) 2024-11-23T18:49:33,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741862_1039 (size=20825) 2024-11-23T18:49:33,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741862_1039 (size=20825) 2024-11-23T18:49:33,947 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/c/aa586dd25db34f6c9526f3e993cd6e68 2024-11-23T18:49:33,955 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/b/b51b16658bb54c82afdc8ba1bcaff6a0 as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/b/b51b16658bb54c82afdc8ba1bcaff6a0 2024-11-23T18:49:33,962 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/b/b51b16658bb54c82afdc8ba1bcaff6a0, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-23T18:49:33,963 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/c/aa586dd25db34f6c9526f3e993cd6e68 as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/c/aa586dd25db34f6c9526f3e993cd6e68 2024-11-23T18:49:33,969 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/c/aa586dd25db34f6c9526f3e993cd6e68, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-23T18:49:33,970 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 45ce04fe4b79f00ca440f4fae21b7a85 in 69ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-23T18:49:33,982 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732387768637/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732387768342.45ce04fe4b79f00ca440f4fae21b7a85. 2024-11-23T18:49:33,985 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/recovered.edits/0000000000000003002 2024-11-23T18:49:33,985 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-23T18:49:33,985 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 45ce04fe4b79f00ca440f4fae21b7a85 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-23T18:49:33,995 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/c/9b4765deedf3433db86cebd98de0b17d is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732387768604/Put/seqid=0 2024-11-23T18:49:34,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741863_1040 (size=50301) 2024-11-23T18:49:34,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741863_1040 (size=50301) 2024-11-23T18:49:34,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741863_1040 (size=50301) 2024-11-23T18:49:34,003 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/c/9b4765deedf3433db86cebd98de0b17d 2024-11-23T18:49:34,009 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9b4765deedf3433db86cebd98de0b17d 2024-11-23T18:49:34,010 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/.tmp/c/9b4765deedf3433db86cebd98de0b17d as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/c/9b4765deedf3433db86cebd98de0b17d 2024-11-23T18:49:34,017 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9b4765deedf3433db86cebd98de0b17d 2024-11-23T18:49:34,017 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/c/9b4765deedf3433db86cebd98de0b17d, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-23T18:49:34,017 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 45ce04fe4b79f00ca440f4fae21b7a85 in 32ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-23T18:49:34,018 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/recovered.edits/0000000000000003002 2024-11-23T18:49:34,019 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:34,019 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:34,020 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T18:49:34,022 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 45ce04fe4b79f00ca440f4fae21b7a85 2024-11-23T18:49:34,025 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenIntoWAL/45ce04fe4b79f00ca440f4fae21b7a85/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-23T18:49:34,027 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 45ce04fe4b79f00ca440f4fae21b7a85; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72726181, jitterRate=0.08370454609394073}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T18:49:34,027 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 45ce04fe4b79f00ca440f4fae21b7a85: Writing region info on filesystem at 1732387772882Initializing all the Stores at 1732387772884 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387772884Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387772884Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387772884Cleaning up temporary data from old regions at 1732387774020 (+1136 ms)Region opened successfully at 1732387774027 (+7 ms) 2024-11-23T18:49:34,104 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 45ce04fe4b79f00ca440f4fae21b7a85, disabling compactions & flushes 2024-11-23T18:49:34,104 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732387768342.45ce04fe4b79f00ca440f4fae21b7a85. 2024-11-23T18:49:34,104 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732387768342.45ce04fe4b79f00ca440f4fae21b7a85. 2024-11-23T18:49:34,104 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732387768342.45ce04fe4b79f00ca440f4fae21b7a85. after waiting 0 ms 2024-11-23T18:49:34,104 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732387768342.45ce04fe4b79f00ca440f4fae21b7a85. 2024-11-23T18:49:34,106 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732387768342.45ce04fe4b79f00ca440f4fae21b7a85. 2024-11-23T18:49:34,106 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 45ce04fe4b79f00ca440f4fae21b7a85: Waiting for close lock at 1732387774104Disabling compacts and flushes for region at 1732387774104Disabling writes for close at 1732387774104Writing region close event to WAL at 1732387774106 (+2 ms)Closed at 1732387774106 2024-11-23T18:49:34,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741857_1034 (size=95) 2024-11-23T18:49:34,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741857_1034 (size=95) 2024-11-23T18:49:34,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741857_1034 (size=95) 2024-11-23T18:49:34,113 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:49:34,114 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732387772862) 2024-11-23T18:49:34,128 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=399 (was 385) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34429 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:43835 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-911535090_22 at /127.0.0.1:54546 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:35915 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-911535090_22 at /127.0.0.1:38294 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:43835 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:34429 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-911535090_22 at /127.0.0.1:56260 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-911535090_22 at /127.0.0.1:54564 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35915 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=306 (was 272) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8989 (was 9054) 2024-11-23T18:49:34,138 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=399, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=306, ProcessCount=11, AvailableMemoryMB=8988 2024-11-23T18:49:34,154 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:34,157 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:34,157 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:49:34,161 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-40070263, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/hregion-40070263, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:34,176 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-40070263/hregion-40070263.1732387774161, exclude list is [], retry=0 2024-11-23T18:49:34,180 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:34,180 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:34,180 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:34,187 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-40070263/hregion-40070263.1732387774161 2024-11-23T18:49:34,187 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:39619:39619)] 2024-11-23T18:49:34,188 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0f7849ed5afbdd951db5ea1117adf666, NAME => 'test2727,,1732387774155.0f7849ed5afbdd951db5ea1117adf666.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43835/hbase 2024-11-23T18:49:34,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741865_1042 (size=43) 2024-11-23T18:49:34,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741865_1042 (size=43) 2024-11-23T18:49:34,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741865_1042 (size=43) 2024-11-23T18:49:34,200 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732387774155.0f7849ed5afbdd951db5ea1117adf666.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:34,202 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:34,203 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f7849ed5afbdd951db5ea1117adf666 columnFamilyName a 2024-11-23T18:49:34,203 DEBUG [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:34,204 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] regionserver.HStore(327): Store=0f7849ed5afbdd951db5ea1117adf666/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:34,204 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:34,205 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f7849ed5afbdd951db5ea1117adf666 columnFamilyName b 2024-11-23T18:49:34,205 DEBUG [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:34,206 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] regionserver.HStore(327): Store=0f7849ed5afbdd951db5ea1117adf666/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:34,206 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:34,208 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f7849ed5afbdd951db5ea1117adf666 columnFamilyName c 2024-11-23T18:49:34,208 DEBUG [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:34,208 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] regionserver.HStore(327): Store=0f7849ed5afbdd951db5ea1117adf666/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:34,208 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:34,209 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:34,210 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:34,211 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:34,211 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:34,212 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:49:34,213 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:34,216 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:49:34,217 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0f7849ed5afbdd951db5ea1117adf666; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65706110, jitterRate=-0.020902663469314575}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:49:34,218 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0f7849ed5afbdd951db5ea1117adf666: Writing region info on filesystem at 1732387774200Initializing all the Stores at 1732387774201 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387774201Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387774201Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387774201Cleaning up temporary data from old regions at 1732387774211 (+10 ms)Region opened successfully at 1732387774218 (+7 ms) 2024-11-23T18:49:34,218 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0f7849ed5afbdd951db5ea1117adf666, disabling compactions & flushes 2024-11-23T18:49:34,218 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732387774155.0f7849ed5afbdd951db5ea1117adf666. 2024-11-23T18:49:34,218 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732387774155.0f7849ed5afbdd951db5ea1117adf666. 2024-11-23T18:49:34,218 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732387774155.0f7849ed5afbdd951db5ea1117adf666. after waiting 0 ms 2024-11-23T18:49:34,218 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732387774155.0f7849ed5afbdd951db5ea1117adf666. 2024-11-23T18:49:34,219 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732387774155.0f7849ed5afbdd951db5ea1117adf666. 2024-11-23T18:49:34,219 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0f7849ed5afbdd951db5ea1117adf666: Waiting for close lock at 1732387774218Disabling compacts and flushes for region at 1732387774218Disabling writes for close at 1732387774218Writing region close event to WAL at 1732387774218Closed at 1732387774218 2024-11-23T18:49:34,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741864_1041 (size=95) 2024-11-23T18:49:34,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741864_1041 (size=95) 2024-11-23T18:49:34,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741864_1041 (size=95) 2024-11-23T18:49:34,225 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:49:34,225 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-40070263:(num 1732387774161) 2024-11-23T18:49:34,225 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:49:34,227 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:34,241 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774228, exclude list is [], retry=0 2024-11-23T18:49:34,244 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:34,244 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:34,244 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:34,247 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774228 2024-11-23T18:49:34,247 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:37451:37451)] 2024-11-23T18:49:34,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741866_1043 (size=263359) 2024-11-23T18:49:34,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741866_1043 (size=263359) 2024-11-23T18:49:34,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741866_1043 (size=263359) 2024-11-23T18:49:34,421 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774228, size=257.2 K (263359bytes) 2024-11-23T18:49:34,422 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774228 2024-11-23T18:49:34,422 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774228 after 0ms 2024-11-23T18:49:34,426 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774228: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:34,427 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774228 took 6ms 2024-11-23T18:49:34,432 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732387774228.temp 2024-11-23T18:49:34,436 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000000001-wal.1732387774228.temp 2024-11-23T18:49:34,494 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774228 so closing down 2024-11-23T18:49:34,494 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:49:34,494 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:49:34,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741867_1044 (size=263359) 2024-11-23T18:49:34,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741867_1044 (size=263359) 2024-11-23T18:49:34,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741867_1044 (size=263359) 2024-11-23T18:49:34,498 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000000001-wal.1732387774228.temp (wrote 3000 edits, skipped 0 edits in 39 ms) 2024-11-23T18:49:34,500 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000000001-wal.1732387774228.temp to hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000003000 2024-11-23T18:49:34,500 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774228, size=257.2 K, length=263359, corrupted=false, cancelled=false 2024-11-23T18:49:34,500 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774228, journal: Splitting hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774228, size=257.2 K (263359bytes) at 1732387774422Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000000001-wal.1732387774228.temp at 1732387774436 (+14 ms)Split 1024 edits, skipped 0 edits. at 1732387774452 (+16 ms)Split 2048 edits, skipped 0 edits. at 1732387774474 (+22 ms)Finishing writing output for hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774228 so closing down at 1732387774494 (+20 ms)3 split writer threads finished at 1732387774494Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000000001-wal.1732387774228.temp (wrote 3000 edits, skipped 0 edits in 39 ms) at 1732387774498 (+4 ms)Rename recovered edits hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000000001-wal.1732387774228.temp to hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000003000 at 1732387774500 (+2 ms)Processed 3000 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774228, size=257.2 K, length=263359, corrupted=false, cancelled=false at 1732387774500 2024-11-23T18:49:34,502 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774228 to hdfs://localhost:43835/hbase/oldWALs/wal.1732387774228 2024-11-23T18:49:34,504 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000003000 2024-11-23T18:49:34,504 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:49:34,506 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:34,527 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774507, exclude list is [], retry=0 2024-11-23T18:49:34,531 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:34,531 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:34,532 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:34,534 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774507 2024-11-23T18:49:34,534 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:39619:39619)] 2024-11-23T18:49:34,598 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T18:49:34,599 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-23T18:49:34,601 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T18:49:34,601 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-23T18:49:34,602 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-23T18:49:34,602 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-23T18:49:34,603 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-23T18:49:34,603 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-23T18:49:34,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741868_1045 (size=263486) 2024-11-23T18:49:34,727 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774507 not finished, retry = 0 2024-11-23T18:49:34,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741868_1045 (size=263486) 2024-11-23T18:49:34,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741868_1045 (size=263486) 2024-11-23T18:49:34,845 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774507, size=257.3 K (263486bytes) 2024-11-23T18:49:34,845 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774507 2024-11-23T18:49:34,846 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774507 after 1ms 2024-11-23T18:49:34,849 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774507: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:34,850 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774507 took 5ms 2024-11-23T18:49:34,855 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732387774507.temp 2024-11-23T18:49:34,856 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000003001-wal.1732387774507.temp 2024-11-23T18:49:34,912 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774507 so closing down 2024-11-23T18:49:34,912 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:49:34,913 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:49:34,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741869_1046 (size=263486) 2024-11-23T18:49:34,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741869_1046 (size=263486) 2024-11-23T18:49:34,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741869_1046 (size=263486) 2024-11-23T18:49:34,916 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000003001-wal.1732387774507.temp (wrote 3000 edits, skipped 0 edits in 37 ms) 2024-11-23T18:49:34,918 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000003001-wal.1732387774507.temp to hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000006000 2024-11-23T18:49:34,918 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 67 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774507, size=257.3 K, length=263486, corrupted=false, cancelled=false 2024-11-23T18:49:34,918 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774507, journal: Splitting hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774507, size=257.3 K (263486bytes) at 1732387774845Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000003001-wal.1732387774507.temp at 1732387774856 (+11 ms)Split 1024 edits, skipped 0 edits. at 1732387774874 (+18 ms)Split 2048 edits, skipped 0 edits. at 1732387774893 (+19 ms)Finishing writing output for hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774507 so closing down at 1732387774912 (+19 ms)3 split writer threads finished at 1732387774913 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000003001-wal.1732387774507.temp (wrote 3000 edits, skipped 0 edits in 37 ms) at 1732387774916 (+3 ms)Rename recovered edits hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000003001-wal.1732387774507.temp to hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000006000 at 1732387774918 (+2 ms)Processed 3000 edits across 1 Regions in 67 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774507, size=257.3 K, length=263486, corrupted=false, cancelled=false at 1732387774918 2024-11-23T18:49:34,920 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774507 to hdfs://localhost:43835/hbase/oldWALs/wal.1732387774507 2024-11-23T18:49:34,921 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000006000 2024-11-23T18:49:34,922 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:49:34,925 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/test2727-manual,16010,1732387774154, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:34,940 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774925, exclude list is [], retry=0 2024-11-23T18:49:34,943 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:34,943 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:34,944 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:34,947 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732387774154/wal.1732387774925 2024-11-23T18:49:34,947 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:37451:37451)] 2024-11-23T18:49:34,947 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0f7849ed5afbdd951db5ea1117adf666, NAME => 'test2727,,1732387774155.0f7849ed5afbdd951db5ea1117adf666.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:34,947 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732387774155.0f7849ed5afbdd951db5ea1117adf666.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:34,948 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:34,948 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:34,950 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:34,951 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f7849ed5afbdd951db5ea1117adf666 columnFamilyName a 2024-11-23T18:49:34,951 DEBUG [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:34,952 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] regionserver.HStore(327): Store=0f7849ed5afbdd951db5ea1117adf666/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:34,952 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:34,954 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f7849ed5afbdd951db5ea1117adf666 columnFamilyName b 2024-11-23T18:49:34,954 DEBUG [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:34,955 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] regionserver.HStore(327): Store=0f7849ed5afbdd951db5ea1117adf666/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:34,955 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:34,957 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f7849ed5afbdd951db5ea1117adf666 columnFamilyName c 2024-11-23T18:49:34,957 DEBUG [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:34,957 INFO [StoreOpener-0f7849ed5afbdd951db5ea1117adf666-1 {}] regionserver.HStore(327): Store=0f7849ed5afbdd951db5ea1117adf666/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:34,958 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:34,959 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:34,961 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:34,962 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000003000 2024-11-23T18:49:34,965 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:35,016 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000003000 2024-11-23T18:49:35,018 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000006000 2024-11-23T18:49:35,020 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:35,068 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000006000 2024-11-23T18:49:35,068 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0f7849ed5afbdd951db5ea1117adf666 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-23T18:49:35,090 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/.tmp/a/f4c532f0182944969a4b1d04d51537f4 is 41, key is test2727/a:100/1732387774541/Put/seqid=0 2024-11-23T18:49:35,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741871_1048 (size=84227) 2024-11-23T18:49:35,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741871_1048 (size=84227) 2024-11-23T18:49:35,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741871_1048 (size=84227) 2024-11-23T18:49:35,099 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/.tmp/a/f4c532f0182944969a4b1d04d51537f4 2024-11-23T18:49:35,125 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/.tmp/b/618cfd7527d048f4a4455424e5f1842b is 41, key is test2727/b:100/1732387774610/Put/seqid=0 2024-11-23T18:49:35,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741872_1049 (size=84609) 2024-11-23T18:49:35,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741872_1049 (size=84609) 2024-11-23T18:49:35,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741872_1049 (size=84609) 2024-11-23T18:49:35,132 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/.tmp/b/618cfd7527d048f4a4455424e5f1842b 2024-11-23T18:49:35,157 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/.tmp/c/caaaa648681840bd9d536b35d7229053 is 41, key is test2727/c:100/1732387774682/Put/seqid=0 2024-11-23T18:49:35,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741873_1050 (size=84609) 2024-11-23T18:49:35,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741873_1050 (size=84609) 2024-11-23T18:49:35,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741873_1050 (size=84609) 2024-11-23T18:49:35,165 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/.tmp/c/caaaa648681840bd9d536b35d7229053 2024-11-23T18:49:35,173 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/.tmp/a/f4c532f0182944969a4b1d04d51537f4 as hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/a/f4c532f0182944969a4b1d04d51537f4 2024-11-23T18:49:35,181 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/a/f4c532f0182944969a4b1d04d51537f4, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-23T18:49:35,183 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/.tmp/b/618cfd7527d048f4a4455424e5f1842b as hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/b/618cfd7527d048f4a4455424e5f1842b 2024-11-23T18:49:35,190 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/b/618cfd7527d048f4a4455424e5f1842b, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-23T18:49:35,191 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/.tmp/c/caaaa648681840bd9d536b35d7229053 as hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/c/caaaa648681840bd9d536b35d7229053 2024-11-23T18:49:35,198 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/c/caaaa648681840bd9d536b35d7229053, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-23T18:49:35,199 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 0f7849ed5afbdd951db5ea1117adf666 in 131ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-23T18:49:35,200 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000003000 2024-11-23T18:49:35,201 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/0000000000000006000 2024-11-23T18:49:35,202 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:35,202 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:35,203 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:49:35,205 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0f7849ed5afbdd951db5ea1117adf666 2024-11-23T18:49:35,208 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/test2727/0f7849ed5afbdd951db5ea1117adf666/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-23T18:49:35,209 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0f7849ed5afbdd951db5ea1117adf666; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69362036, jitterRate=0.03357487916946411}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:49:35,210 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0f7849ed5afbdd951db5ea1117adf666: Writing region info on filesystem at 1732387774948Initializing all the Stores at 1732387774949 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387774949Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387774949Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387774949Obtaining lock to block concurrent updates at 1732387775068 (+119 ms)Preparing flush snapshotting stores in 0f7849ed5afbdd951db5ea1117adf666 at 1732387775068Finished memstore snapshotting test2727,,1732387774155.0f7849ed5afbdd951db5ea1117adf666., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732387775068Flushing stores of test2727,,1732387774155.0f7849ed5afbdd951db5ea1117adf666. at 1732387775068Flushing 0f7849ed5afbdd951db5ea1117adf666/a: creating writer at 1732387775068Flushing 0f7849ed5afbdd951db5ea1117adf666/a: appending metadata at 1732387775089 (+21 ms)Flushing 0f7849ed5afbdd951db5ea1117adf666/a: closing flushed file at 1732387775089Flushing 0f7849ed5afbdd951db5ea1117adf666/b: creating writer at 1732387775106 (+17 ms)Flushing 0f7849ed5afbdd951db5ea1117adf666/b: appending metadata at 1732387775123 (+17 ms)Flushing 0f7849ed5afbdd951db5ea1117adf666/b: closing flushed file at 1732387775123Flushing 0f7849ed5afbdd951db5ea1117adf666/c: creating writer at 1732387775138 (+15 ms)Flushing 0f7849ed5afbdd951db5ea1117adf666/c: appending metadata at 1732387775157 (+19 ms)Flushing 0f7849ed5afbdd951db5ea1117adf666/c: closing flushed file at 1732387775157Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@143ae945: reopening flushed file at 1732387775172 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13a1547d: reopening flushed file at 1732387775182 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d20b104: reopening flushed file at 1732387775190 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 0f7849ed5afbdd951db5ea1117adf666 in 131ms, sequenceid=6000, compaction requested=false; wal=null at 1732387775199 (+9 ms)Cleaning up temporary data from old regions at 1732387775202 (+3 ms)Region opened successfully at 1732387775210 (+8 ms) 2024-11-23T18:49:35,211 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-23T18:49:35,212 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0f7849ed5afbdd951db5ea1117adf666, disabling compactions & flushes 2024-11-23T18:49:35,212 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732387774155.0f7849ed5afbdd951db5ea1117adf666. 2024-11-23T18:49:35,212 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732387774155.0f7849ed5afbdd951db5ea1117adf666. 2024-11-23T18:49:35,212 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732387774155.0f7849ed5afbdd951db5ea1117adf666. after waiting 0 ms 2024-11-23T18:49:35,212 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732387774155.0f7849ed5afbdd951db5ea1117adf666. 2024-11-23T18:49:35,213 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732387774155.0f7849ed5afbdd951db5ea1117adf666. 2024-11-23T18:49:35,213 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0f7849ed5afbdd951db5ea1117adf666: Waiting for close lock at 1732387775211Disabling compacts and flushes for region at 1732387775211Disabling writes for close at 1732387775212 (+1 ms)Writing region close event to WAL at 1732387775213 (+1 ms)Closed at 1732387775213 2024-11-23T18:49:35,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741870_1047 (size=95) 2024-11-23T18:49:35,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741870_1047 (size=95) 2024-11-23T18:49:35,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741870_1047 (size=95) 2024-11-23T18:49:35,219 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:49:35,219 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732387774925) 2024-11-23T18:49:35,233 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=401 (was 399) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:38366 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:56388 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:54692 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=306 (was 306), ProcessCount=11 (was 11), AvailableMemoryMB=8780 (was 8988) 2024-11-23T18:49:35,244 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=401, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=306, ProcessCount=11, AvailableMemoryMB=8778 2024-11-23T18:49:35,262 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:35,268 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387775261, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:35,269 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732387775269 2024-11-23T18:49:35,276 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387775261/wal.1732387775269 2024-11-23T18:49:35,279 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:37451:37451)] 2024-11-23T18:49:35,280 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d10511464e58f56759296544989b396e, NAME => 'testSequentialEditLogSeqNum,,1732387775262.d10511464e58f56759296544989b396e.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:35,280 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732387775262.d10511464e58f56759296544989b396e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:35,281 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d10511464e58f56759296544989b396e 2024-11-23T18:49:35,281 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d10511464e58f56759296544989b396e 2024-11-23T18:49:35,282 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:43835/hbase/data/default/testSequentialEditLogSeqNum/d10511464e58f56759296544989b396e doesn't exist for region: d10511464e58f56759296544989b396e on table testSequentialEditLogSeqNum 2024-11-23T18:49:35,282 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: d10511464e58f56759296544989b396e on table testSequentialEditLogSeqNum 2024-11-23T18:49:35,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741875_1052 (size=62) 2024-11-23T18:49:35,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741875_1052 (size=62) 2024-11-23T18:49:35,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741875_1052 (size=62) 2024-11-23T18:49:35,296 INFO [StoreOpener-d10511464e58f56759296544989b396e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d10511464e58f56759296544989b396e 2024-11-23T18:49:35,298 INFO [StoreOpener-d10511464e58f56759296544989b396e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d10511464e58f56759296544989b396e columnFamilyName a 2024-11-23T18:49:35,298 DEBUG [StoreOpener-d10511464e58f56759296544989b396e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:35,298 INFO [StoreOpener-d10511464e58f56759296544989b396e-1 {}] regionserver.HStore(327): Store=d10511464e58f56759296544989b396e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:35,298 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d10511464e58f56759296544989b396e 2024-11-23T18:49:35,299 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testSequentialEditLogSeqNum/d10511464e58f56759296544989b396e 2024-11-23T18:49:35,299 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testSequentialEditLogSeqNum/d10511464e58f56759296544989b396e 2024-11-23T18:49:35,300 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d10511464e58f56759296544989b396e 2024-11-23T18:49:35,300 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d10511464e58f56759296544989b396e 2024-11-23T18:49:35,301 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d10511464e58f56759296544989b396e 2024-11-23T18:49:35,304 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testSequentialEditLogSeqNum/d10511464e58f56759296544989b396e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:49:35,305 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d10511464e58f56759296544989b396e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63418647, jitterRate=-0.054988518357276917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:49:35,306 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d10511464e58f56759296544989b396e: Writing region info on filesystem at 1732387775281Initializing all the Stores at 1732387775296 (+15 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387775296Cleaning up temporary data from old regions at 1732387775300 (+4 ms)Region opened successfully at 1732387775306 (+6 ms) 2024-11-23T18:49:35,321 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d10511464e58f56759296544989b396e 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-23T18:49:35,351 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testSequentialEditLogSeqNum/d10511464e58f56759296544989b396e/.tmp/a/9ea036c9275b4fea9013c64861e7ea99 is 81, key is testSequentialEditLogSeqNum/a:x0/1732387775306/Put/seqid=0 2024-11-23T18:49:35,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741876_1053 (size=5833) 2024-11-23T18:49:35,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741876_1053 (size=5833) 2024-11-23T18:49:35,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741876_1053 (size=5833) 2024-11-23T18:49:35,360 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testSequentialEditLogSeqNum/d10511464e58f56759296544989b396e/.tmp/a/9ea036c9275b4fea9013c64861e7ea99 2024-11-23T18:49:35,368 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testSequentialEditLogSeqNum/d10511464e58f56759296544989b396e/.tmp/a/9ea036c9275b4fea9013c64861e7ea99 as hdfs://localhost:43835/hbase/data/default/testSequentialEditLogSeqNum/d10511464e58f56759296544989b396e/a/9ea036c9275b4fea9013c64861e7ea99 2024-11-23T18:49:35,375 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testSequentialEditLogSeqNum/d10511464e58f56759296544989b396e/a/9ea036c9275b4fea9013c64861e7ea99, entries=10, sequenceid=13, filesize=5.7 K 2024-11-23T18:49:35,377 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for d10511464e58f56759296544989b396e in 57ms, sequenceid=13, compaction requested=false 2024-11-23T18:49:35,377 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d10511464e58f56759296544989b396e: 2024-11-23T18:49:35,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T18:49:35,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T18:49:35,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T18:49:35,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T18:49:35,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T18:49:35,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741874_1051 (size=1844) 2024-11-23T18:49:35,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741874_1051 (size=1844) 2024-11-23T18:49:35,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741874_1051 (size=1844) 2024-11-23T18:49:35,427 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43835/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387775261/wal.1732387775269, size=1.8 K (1844bytes) 2024-11-23T18:49:35,427 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387775261/wal.1732387775269 2024-11-23T18:49:35,428 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43835/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387775261/wal.1732387775269 after 1ms 2024-11-23T18:49:35,431 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387775261/wal.1732387775269: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:35,432 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43835/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387775261/wal.1732387775269 took 5ms 2024-11-23T18:49:35,439 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43835/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387775261/wal.1732387775269 so closing down 2024-11-23T18:49:35,439 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:49:35,441 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732387775269.temp 2024-11-23T18:49:35,443 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testSequentialEditLogSeqNum/d10511464e58f56759296544989b396e/recovered.edits/0000000000000000003-wal.1732387775269.temp 2024-11-23T18:49:35,444 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:49:35,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741877_1054 (size=1477) 2024-11-23T18:49:35,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741877_1054 (size=1477) 2024-11-23T18:49:35,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741877_1054 (size=1477) 2024-11-23T18:49:35,473 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testSequentialEditLogSeqNum/d10511464e58f56759296544989b396e/recovered.edits/0000000000000000003-wal.1732387775269.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-23T18:49:35,475 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43835/hbase/data/default/testSequentialEditLogSeqNum/d10511464e58f56759296544989b396e/recovered.edits/0000000000000000003-wal.1732387775269.temp to hdfs://localhost:43835/hbase/data/default/testSequentialEditLogSeqNum/d10511464e58f56759296544989b396e/recovered.edits/0000000000000000020 2024-11-23T18:49:35,475 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 39 ms; skipped=2; WAL=hdfs://localhost:43835/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387775261/wal.1732387775269, size=1.8 K, length=1844, corrupted=false, cancelled=false 2024-11-23T18:49:35,475 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43835/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387775261/wal.1732387775269, journal: Splitting hdfs://localhost:43835/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387775261/wal.1732387775269, size=1.8 K (1844bytes) at 1732387775427Finishing writing output for hdfs://localhost:43835/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387775261/wal.1732387775269 so closing down at 1732387775439 (+12 ms)Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testSequentialEditLogSeqNum/d10511464e58f56759296544989b396e/recovered.edits/0000000000000000003-wal.1732387775269.temp at 1732387775443 (+4 ms)3 split writer threads finished at 1732387775444 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testSequentialEditLogSeqNum/d10511464e58f56759296544989b396e/recovered.edits/0000000000000000003-wal.1732387775269.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732387775473 (+29 ms)Rename recovered edits hdfs://localhost:43835/hbase/data/default/testSequentialEditLogSeqNum/d10511464e58f56759296544989b396e/recovered.edits/0000000000000000003-wal.1732387775269.temp to hdfs://localhost:43835/hbase/data/default/testSequentialEditLogSeqNum/d10511464e58f56759296544989b396e/recovered.edits/0000000000000000020 at 1732387775475 (+2 ms)Processed 17 edits across 1 Regions in 39 ms; skipped=2; WAL=hdfs://localhost:43835/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387775261/wal.1732387775269, size=1.8 K, length=1844, corrupted=false, cancelled=false at 1732387775475 2024-11-23T18:49:35,497 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=408 (was 401) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:38366 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:56388 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:54692 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=306 (was 306), ProcessCount=11 (was 11), AvailableMemoryMB=8698 (was 8778) 2024-11-23T18:49:35,517 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=408, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=306, ProcessCount=11, AvailableMemoryMB=8696 2024-11-23T18:49:35,534 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:35,537 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:35,568 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:49:35,572 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-53608574, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/hregion-53608574, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:35,591 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-53608574/hregion-53608574.1732387775573, exclude list is [], retry=0 2024-11-23T18:49:35,595 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:35,595 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:35,596 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:35,599 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-53608574/hregion-53608574.1732387775573 2024-11-23T18:49:35,600 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:37451:37451)] 2024-11-23T18:49:35,600 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c96b7716c7c81c47b97dec4109a31223, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43835/hbase 2024-11-23T18:49:35,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741879_1056 (size=70) 2024-11-23T18:49:35,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741879_1056 (size=70) 2024-11-23T18:49:35,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741879_1056 (size=70) 2024-11-23T18:49:35,625 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:35,629 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,632 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c96b7716c7c81c47b97dec4109a31223 columnFamilyName a 2024-11-23T18:49:35,632 DEBUG [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:35,633 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(327): Store=c96b7716c7c81c47b97dec4109a31223/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:35,633 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,636 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c96b7716c7c81c47b97dec4109a31223 columnFamilyName b 2024-11-23T18:49:35,636 DEBUG [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:35,637 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(327): Store=c96b7716c7c81c47b97dec4109a31223/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:35,637 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,639 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c96b7716c7c81c47b97dec4109a31223 columnFamilyName c 2024-11-23T18:49:35,640 DEBUG [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:35,640 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(327): Store=c96b7716c7c81c47b97dec4109a31223/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:35,641 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,642 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,643 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,644 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,644 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,645 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:49:35,647 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,650 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:49:35,650 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c96b7716c7c81c47b97dec4109a31223; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64791989, jitterRate=-0.03452412784099579}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:49:35,651 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c96b7716c7c81c47b97dec4109a31223: Writing region info on filesystem at 1732387775625Initializing all the Stores at 1732387775629 (+4 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387775629Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387775629Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387775629Cleaning up temporary data from old regions at 1732387775644 (+15 ms)Region opened successfully at 1732387775651 (+7 ms) 2024-11-23T18:49:35,652 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c96b7716c7c81c47b97dec4109a31223, disabling compactions & flushes 2024-11-23T18:49:35,652 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223. 2024-11-23T18:49:35,652 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223. 2024-11-23T18:49:35,652 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223. after waiting 0 ms 2024-11-23T18:49:35,652 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223. 2024-11-23T18:49:35,653 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223. 2024-11-23T18:49:35,653 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c96b7716c7c81c47b97dec4109a31223: Waiting for close lock at 1732387775652Disabling compacts and flushes for region at 1732387775652Disabling writes for close at 1732387775652Writing region close event to WAL at 1732387775653 (+1 ms)Closed at 1732387775653 2024-11-23T18:49:35,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741878_1055 (size=95) 2024-11-23T18:49:35,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741878_1055 (size=95) 2024-11-23T18:49:35,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741878_1055 (size=95) 2024-11-23T18:49:35,661 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:49:35,661 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-53608574:(num 1732387775573) 2024-11-23T18:49:35,661 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:49:35,664 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:35,685 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664, exclude list is [], retry=0 2024-11-23T18:49:35,688 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:35,688 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:35,689 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:35,691 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664 2024-11-23T18:49:35,691 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:43381:43381)] 2024-11-23T18:49:35,692 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c96b7716c7c81c47b97dec4109a31223, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:35,692 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:35,692 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,692 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,696 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,698 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c96b7716c7c81c47b97dec4109a31223 columnFamilyName a 2024-11-23T18:49:35,698 DEBUG [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:35,699 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(327): Store=c96b7716c7c81c47b97dec4109a31223/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:35,699 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,700 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c96b7716c7c81c47b97dec4109a31223 columnFamilyName b 2024-11-23T18:49:35,700 DEBUG [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:35,700 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(327): Store=c96b7716c7c81c47b97dec4109a31223/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:35,701 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,702 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c96b7716c7c81c47b97dec4109a31223 columnFamilyName c 2024-11-23T18:49:35,702 DEBUG [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:35,702 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(327): Store=c96b7716c7c81c47b97dec4109a31223/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:35,702 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,703 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,704 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,705 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,705 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,706 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:49:35,707 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:35,708 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c96b7716c7c81c47b97dec4109a31223; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61619488, jitterRate=-0.08179807662963867}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:49:35,709 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c96b7716c7c81c47b97dec4109a31223: Writing region info on filesystem at 1732387775692Initializing all the Stores at 1732387775694 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387775694Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387775696 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387775696Cleaning up temporary data from old regions at 1732387775705 (+9 ms)Region opened successfully at 1732387775709 (+4 ms) 2024-11-23T18:49:35,713 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732387775712/Put/seqid=0 2024-11-23T18:49:35,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741881_1058 (size=4826) 2024-11-23T18:49:35,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741881_1058 (size=4826) 2024-11-23T18:49:35,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741881_1058 (size=4826) 2024-11-23T18:49:35,720 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43835/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in c96b7716c7c81c47b97dec4109a31223/a 2024-11-23T18:49:35,728 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-23T18:49:35,728 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-23T18:49:35,728 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c96b7716c7c81c47b97dec4109a31223: 2024-11-23T18:49:35,730 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/a/86f3f8018a4b4b9d95e5a19a52f5d369_SeqId_3_ 2024-11-23T18:49:35,731 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43835/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into c96b7716c7c81c47b97dec4109a31223/a as hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/a/86f3f8018a4b4b9d95e5a19a52f5d369_SeqId_3_ - updating store file list. 2024-11-23T18:49:35,737 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 86f3f8018a4b4b9d95e5a19a52f5d369_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-23T18:49:35,737 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/a/86f3f8018a4b4b9d95e5a19a52f5d369_SeqId_3_ into c96b7716c7c81c47b97dec4109a31223/a 2024-11-23T18:49:35,738 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43835/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into c96b7716c7c81c47b97dec4109a31223/a (new location: hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/a/86f3f8018a4b4b9d95e5a19a52f5d369_SeqId_3_) 2024-11-23T18:49:35,776 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43835/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664, size=0 (0bytes) 2024-11-23T18:49:35,776 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43835/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664 might be still open, length is 0 2024-11-23T18:49:35,776 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664 2024-11-23T18:49:35,777 WARN [IPC Server handler 4 on default port 43835 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-23T18:49:35,777 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43835/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664 after 1ms 2024-11-23T18:49:38,320 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:56472 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37657:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56472 dst: /127.0.0.1:37657 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37657 remote=/127.0.0.1:56472]. Total timeout mills is 60000, 57423 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:38,320 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:54770 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40877:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54770 dst: /127.0.0.1:40877 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:38,321 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:38482 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43523:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38482 dst: /127.0.0.1:43523 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:38,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741880_1059 (size=474) 2024-11-23T18:49:38,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741880_1059 (size=474) 2024-11-23T18:49:38,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741880_1059 (size=474) 2024-11-23T18:49:39,778 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43835/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664 after 4002ms 2024-11-23T18:49:39,783 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:39,783 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43835/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664 took 4007ms 2024-11-23T18:49:39,785 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43835/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664; continuing. 2024-11-23T18:49:39,785 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43835/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664 so closing down 2024-11-23T18:49:39,785 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:49:39,787 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732387775664.temp 2024-11-23T18:49:39,789 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/recovered.edits/0000000000000000005-wal.1732387775664.temp 2024-11-23T18:49:39,789 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:49:39,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741882_1060 (size=259) 2024-11-23T18:49:39,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741882_1060 (size=259) 2024-11-23T18:49:39,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741882_1060 (size=259) 2024-11-23T18:49:39,799 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/recovered.edits/0000000000000000005-wal.1732387775664.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-23T18:49:39,800 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/recovered.edits/0000000000000000005-wal.1732387775664.temp to hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/recovered.edits/0000000000000000005 2024-11-23T18:49:39,800 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 16 ms; skipped=1; WAL=hdfs://localhost:43835/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664, size=0, length=0, corrupted=false, cancelled=false 2024-11-23T18:49:39,800 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43835/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664, journal: Splitting hdfs://localhost:43835/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664, size=0 (0bytes) at 1732387775776Finishing writing output for hdfs://localhost:43835/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664 so closing down at 1732387779785 (+4009 ms)Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/recovered.edits/0000000000000000005-wal.1732387775664.temp at 1732387779789 (+4 ms)3 split writer threads finished at 1732387779789Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/recovered.edits/0000000000000000005-wal.1732387775664.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732387779799 (+10 ms)Rename recovered edits hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/recovered.edits/0000000000000000005-wal.1732387775664.temp to hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/recovered.edits/0000000000000000005 at 1732387779800 (+1 ms)Processed 2 edits across 1 Regions in 16 ms; skipped=1; WAL=hdfs://localhost:43835/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664, size=0, length=0, corrupted=false, cancelled=false at 1732387779800 2024-11-23T18:49:39,802 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43835/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664 to hdfs://localhost:43835/hbase/oldWALs/wal.1732387775664 2024-11-23T18:49:39,803 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/recovered.edits/0000000000000000005 2024-11-23T18:49:39,803 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:49:39,804 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:39,819 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387779805, exclude list is [], retry=0 2024-11-23T18:49:39,822 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:39,822 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:39,823 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:39,824 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387779805 2024-11-23T18:49:39,824 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:43381:43381)] 2024-11-23T18:49:39,825 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c96b7716c7c81c47b97dec4109a31223, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:39,825 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:39,825 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:39,825 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:39,827 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:39,827 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c96b7716c7c81c47b97dec4109a31223 columnFamilyName a 2024-11-23T18:49:39,828 DEBUG [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:39,834 DEBUG [StoreFileOpener-c96b7716c7c81c47b97dec4109a31223-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 86f3f8018a4b4b9d95e5a19a52f5d369_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-23T18:49:39,834 DEBUG [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/a/86f3f8018a4b4b9d95e5a19a52f5d369_SeqId_3_ 2024-11-23T18:49:39,835 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(327): Store=c96b7716c7c81c47b97dec4109a31223/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:39,835 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:39,836 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c96b7716c7c81c47b97dec4109a31223 columnFamilyName b 2024-11-23T18:49:39,836 DEBUG [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:39,836 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(327): Store=c96b7716c7c81c47b97dec4109a31223/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:39,836 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:39,837 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c96b7716c7c81c47b97dec4109a31223 columnFamilyName c 2024-11-23T18:49:39,837 DEBUG [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:39,837 INFO [StoreOpener-c96b7716c7c81c47b97dec4109a31223-1 {}] regionserver.HStore(327): Store=c96b7716c7c81c47b97dec4109a31223/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:39,837 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:39,838 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:39,840 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:39,840 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/recovered.edits/0000000000000000005 2024-11-23T18:49:39,842 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:39,843 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/recovered.edits/0000000000000000005 2024-11-23T18:49:39,843 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c96b7716c7c81c47b97dec4109a31223 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-23T18:49:39,858 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/.tmp/a/ab7a254d42ec49e996f36cd954af364a is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732387775742/Put/seqid=0 2024-11-23T18:49:39,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741884_1062 (size=5149) 2024-11-23T18:49:39,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741884_1062 (size=5149) 2024-11-23T18:49:39,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741884_1062 (size=5149) 2024-11-23T18:49:39,865 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/.tmp/a/ab7a254d42ec49e996f36cd954af364a 2024-11-23T18:49:39,871 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/.tmp/a/ab7a254d42ec49e996f36cd954af364a as hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/a/ab7a254d42ec49e996f36cd954af364a 2024-11-23T18:49:39,878 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/a/ab7a254d42ec49e996f36cd954af364a, entries=1, sequenceid=5, filesize=5.0 K 2024-11-23T18:49:39,879 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for c96b7716c7c81c47b97dec4109a31223 in 36ms, sequenceid=5, compaction requested=false; wal=null 2024-11-23T18:49:39,880 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/recovered.edits/0000000000000000005 2024-11-23T18:49:39,881 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:39,881 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:39,882 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:49:39,883 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c96b7716c7c81c47b97dec4109a31223 2024-11-23T18:49:39,885 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c96b7716c7c81c47b97dec4109a31223/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-23T18:49:39,886 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c96b7716c7c81c47b97dec4109a31223; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62519212, jitterRate=-0.06839114427566528}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:49:39,886 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c96b7716c7c81c47b97dec4109a31223: Writing region info on filesystem at 1732387779825Initializing all the Stores at 1732387779826 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387779826Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387779826Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387779826Obtaining lock to block concurrent updates at 1732387779843 (+17 ms)Preparing flush snapshotting stores in c96b7716c7c81c47b97dec4109a31223 at 1732387779843Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732387779844 (+1 ms)Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223. at 1732387779844Flushing c96b7716c7c81c47b97dec4109a31223/a: creating writer at 1732387779844Flushing c96b7716c7c81c47b97dec4109a31223/a: appending metadata at 1732387779857 (+13 ms)Flushing c96b7716c7c81c47b97dec4109a31223/a: closing flushed file at 1732387779857Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60fa734f: reopening flushed file at 1732387779870 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for c96b7716c7c81c47b97dec4109a31223 in 36ms, sequenceid=5, compaction requested=false; wal=null at 1732387779879 (+9 ms)Cleaning up temporary data from old regions at 1732387779881 (+2 ms)Region opened successfully at 1732387779886 (+5 ms) 2024-11-23T18:49:39,891 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c96b7716c7c81c47b97dec4109a31223, disabling compactions & flushes 2024-11-23T18:49:39,891 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223. 2024-11-23T18:49:39,891 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223. 2024-11-23T18:49:39,892 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223. after waiting 0 ms 2024-11-23T18:49:39,892 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223. 2024-11-23T18:49:39,892 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732387775535.c96b7716c7c81c47b97dec4109a31223. 2024-11-23T18:49:39,892 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c96b7716c7c81c47b97dec4109a31223: Waiting for close lock at 1732387779891Disabling compacts and flushes for region at 1732387779891Disabling writes for close at 1732387779892 (+1 ms)Writing region close event to WAL at 1732387779892Closed at 1732387779892 2024-11-23T18:49:39,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741883_1061 (size=95) 2024-11-23T18:49:39,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741883_1061 (size=95) 2024-11-23T18:49:39,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741883_1061 (size=95) 2024-11-23T18:49:39,897 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:49:39,897 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732387779805) 2024-11-23T18:49:39,911 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=412 (was 408) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:43835 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1207037913_22 at /127.0.0.1:56518 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1207037913_22 at /127.0.0.1:38518 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1207037913_22 at /127.0.0.1:54792 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:43835 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=282 (was 306), ProcessCount=11 (was 11), AvailableMemoryMB=8697 (was 8696) - AvailableMemoryMB LEAK? - 2024-11-23T18:49:39,923 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=412, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=282, ProcessCount=11, AvailableMemoryMB=8695 2024-11-23T18:49:39,944 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:39,950 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T18:49:39,955 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2eda60943ea1,37117,1732387764258 2024-11-23T18:49:39,957 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b313a6c 2024-11-23T18:49:39,958 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T18:49:39,960 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T18:49:39,964 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T18:49:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-23T18:49:39,973 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T18:49:39,976 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-23T18:49:39,976 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:39,978 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T18:49:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T18:49:39,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741885_1063 (size=694) 2024-11-23T18:49:39,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741885_1063 (size=694) 2024-11-23T18:49:39,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741885_1063 (size=694) 2024-11-23T18:49:39,992 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aafadebc54e468784234072b47bbd24c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe 2024-11-23T18:49:39,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741886_1064 (size=77) 2024-11-23T18:49:40,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741886_1064 (size=77) 2024-11-23T18:49:40,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741886_1064 (size=77) 2024-11-23T18:49:40,001 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:40,001 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing aafadebc54e468784234072b47bbd24c, disabling compactions & flushes 2024-11-23T18:49:40,001 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:40,001 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:40,002 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. after waiting 0 ms 2024-11-23T18:49:40,002 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:40,002 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:40,002 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for aafadebc54e468784234072b47bbd24c: Waiting for close lock at 1732387780001Disabling compacts and flushes for region at 1732387780001Disabling writes for close at 1732387780002 (+1 ms)Writing region close event to WAL at 1732387780002Closed at 1732387780002 2024-11-23T18:49:40,003 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T18:49:40,008 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732387780004"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732387780004"}]},"ts":"1732387780004"} 2024-11-23T18:49:40,011 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T18:49:40,013 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T18:49:40,016 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732387780013"}]},"ts":"1732387780013"} 2024-11-23T18:49:40,020 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-23T18:49:40,021 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {2eda60943ea1=0} racks are {/default-rack=0} 2024-11-23T18:49:40,022 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T18:49:40,022 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T18:49:40,022 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-23T18:49:40,022 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T18:49:40,022 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T18:49:40,022 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-23T18:49:40,022 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T18:49:40,022 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T18:49:40,022 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-23T18:49:40,022 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T18:49:40,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, ASSIGN}] 2024-11-23T18:49:40,026 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, ASSIGN 2024-11-23T18:49:40,027 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, ASSIGN; state=OFFLINE, location=2eda60943ea1,32913,1732387765000; forceNewPlan=false, retain=false 2024-11-23T18:49:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T18:49:40,180 INFO [2eda60943ea1:37117 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T18:49:40,182 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=aafadebc54e468784234072b47bbd24c, regionState=OPENING, regionLocation=2eda60943ea1,32913,1732387765000 2024-11-23T18:49:40,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, ASSIGN because future has completed 2024-11-23T18:49:40,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,32913,1732387765000}] 2024-11-23T18:49:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T18:49:40,352 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:40,353 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => aafadebc54e468784234072b47bbd24c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:40,353 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:40,353 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:40,353 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:40,354 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:40,356 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:40,359 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aafadebc54e468784234072b47bbd24c columnFamilyName cf1 2024-11-23T18:49:40,359 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:40,360 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(327): Store=aafadebc54e468784234072b47bbd24c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:40,361 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:40,364 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aafadebc54e468784234072b47bbd24c columnFamilyName cf2 2024-11-23T18:49:40,364 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:40,365 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(327): Store=aafadebc54e468784234072b47bbd24c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:40,366 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:40,367 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:40,368 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:40,369 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:40,369 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:40,370 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-23T18:49:40,371 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:40,373 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:49:40,374 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened aafadebc54e468784234072b47bbd24c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60857505, jitterRate=-0.09315250813961029}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-23T18:49:40,374 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:40,375 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for aafadebc54e468784234072b47bbd24c: Running coprocessor pre-open hook at 1732387780354Writing region info on filesystem at 1732387780354Initializing all the Stores at 1732387780355 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387780355Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387780356 (+1 ms)Cleaning up temporary data from old regions at 1732387780369 (+13 ms)Running coprocessor post-open hooks at 1732387780374 (+5 ms)Region opened successfully at 1732387780375 (+1 ms) 2024-11-23T18:49:40,377 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., pid=6, masterSystemTime=1732387780342 2024-11-23T18:49:40,379 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:40,379 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:40,380 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=aafadebc54e468784234072b47bbd24c, regionState=OPEN, openSeqNum=2, regionLocation=2eda60943ea1,32913,1732387765000 2024-11-23T18:49:40,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,32913,1732387765000 because future has completed 2024-11-23T18:49:40,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T18:49:40,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,32913,1732387765000 in 197 msec 2024-11-23T18:49:40,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T18:49:40,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, ASSIGN in 364 msec 2024-11-23T18:49:40,393 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T18:49:40,393 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732387780393"}]},"ts":"1732387780393"} 2024-11-23T18:49:40,396 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-23T18:49:40,397 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T18:49:40,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 431 msec 2024-11-23T18:49:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T18:49:40,617 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-23T18:49:40,617 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-23T18:49:40,618 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T18:49:40,625 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-23T18:49:40,625 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T18:49:40,626 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-23T18:49:40,640 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,32913,1732387765000, seqNum=2] 2024-11-23T18:49:40,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=aafadebc54e468784234072b47bbd24c, source=2eda60943ea1,32913,1732387765000, destination=2eda60943ea1,43797,1732387765099, warming up region on 2eda60943ea1,43797,1732387765099 2024-11-23T18:49:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T18:49:40,664 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=aafadebc54e468784234072b47bbd24c, source=2eda60943ea1,32913,1732387765000, destination=2eda60943ea1,43797,1732387765099, running balancer 2024-11-23T18:49:40,666 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56445, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T18:49:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, REOPEN/MOVE 2024-11-23T18:49:40,666 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, REOPEN/MOVE 2024-11-23T18:49:40,669 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=aafadebc54e468784234072b47bbd24c, regionState=CLOSING, regionLocation=2eda60943ea1,32913,1732387765000 2024-11-23T18:49:40,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43797 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:40,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43797 {}] regionserver.HRegion(7855): Warmup {ENCODED => aafadebc54e468784234072b47bbd24c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43797 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:40,671 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:40,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, REOPEN/MOVE because future has completed 2024-11-23T18:49:40,672 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aafadebc54e468784234072b47bbd24c columnFamilyName cf1 2024-11-23T18:49:40,673 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:40,673 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T18:49:40,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,32913,1732387765000}] 2024-11-23T18:49:40,676 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(327): Store=aafadebc54e468784234072b47bbd24c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:40,676 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:40,678 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aafadebc54e468784234072b47bbd24c columnFamilyName cf2 2024-11-23T18:49:40,678 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:40,679 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(327): Store=aafadebc54e468784234072b47bbd24c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43797 {}] regionserver.HRegion(1722): Closing aafadebc54e468784234072b47bbd24c, disabling compactions & flushes 2024-11-23T18:49:40,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43797 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43797 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43797 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. after waiting 0 ms 2024-11-23T18:49:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43797 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:40,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43797 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43797 {}] regionserver.HRegion(1676): Region close journal for aafadebc54e468784234072b47bbd24c: Waiting for close lock at 1732387780679Disabling compacts and flushes for region at 1732387780679Disabling writes for close at 1732387780679Writing region close event to WAL at 1732387780680 (+1 ms)Closed at 1732387780680 2024-11-23T18:49:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-23T18:49:40,833 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:40,834 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T18:49:40,834 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing aafadebc54e468784234072b47bbd24c, disabling compactions & flushes 2024-11-23T18:49:40,834 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:40,834 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:40,834 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. after waiting 0 ms 2024-11-23T18:49:40,834 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:40,834 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing aafadebc54e468784234072b47bbd24c 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-23T18:49:40,851 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/.tmp/cf1/548a1e14ded3450b96f2965ee6e0a2cf is 35, key is r1/cf1:q/1732387780643/Put/seqid=0 2024-11-23T18:49:40,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741887_1065 (size=4783) 2024-11-23T18:49:40,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741887_1065 (size=4783) 2024-11-23T18:49:40,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741887_1065 (size=4783) 2024-11-23T18:49:41,266 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/.tmp/cf1/548a1e14ded3450b96f2965ee6e0a2cf 2024-11-23T18:49:41,281 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/.tmp/cf1/548a1e14ded3450b96f2965ee6e0a2cf as hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/548a1e14ded3450b96f2965ee6e0a2cf 2024-11-23T18:49:41,288 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/548a1e14ded3450b96f2965ee6e0a2cf, entries=1, sequenceid=5, filesize=4.7 K 2024-11-23T18:49:41,290 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for aafadebc54e468784234072b47bbd24c in 455ms, sequenceid=5, compaction requested=false 2024-11-23T18:49:41,290 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-23T18:49:41,295 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-23T18:49:41,297 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:41,298 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for aafadebc54e468784234072b47bbd24c: Waiting for close lock at 1732387780834Running coprocessor pre-close hooks at 1732387780834Disabling compacts and flushes for region at 1732387780834Disabling writes for close at 1732387780834Obtaining lock to block concurrent updates at 1732387780834Preparing flush snapshotting stores in aafadebc54e468784234072b47bbd24c at 1732387780834Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732387780835 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. at 1732387780835Flushing aafadebc54e468784234072b47bbd24c/cf1: creating writer at 1732387780836 (+1 ms)Flushing aafadebc54e468784234072b47bbd24c/cf1: appending metadata at 1732387780851 (+15 ms)Flushing aafadebc54e468784234072b47bbd24c/cf1: closing flushed file at 1732387780851Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68d60ac2: reopening flushed file at 1732387781279 (+428 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for aafadebc54e468784234072b47bbd24c in 455ms, sequenceid=5, compaction requested=false at 1732387781290 (+11 ms)Writing region close event to WAL at 1732387781291 (+1 ms)Running coprocessor post-close hooks at 1732387781296 (+5 ms)Closed at 1732387781297 (+1 ms) 2024-11-23T18:49:41,298 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding aafadebc54e468784234072b47bbd24c move to 2eda60943ea1,43797,1732387765099 record at close sequenceid=5 2024-11-23T18:49:41,301 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:41,302 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=aafadebc54e468784234072b47bbd24c, regionState=CLOSED 2024-11-23T18:49:41,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,32913,1732387765000 because future has completed 2024-11-23T18:49:41,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T18:49:41,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,32913,1732387765000 in 633 msec 2024-11-23T18:49:41,311 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, REOPEN/MOVE; state=CLOSED, location=2eda60943ea1,43797,1732387765099; forceNewPlan=false, retain=false 2024-11-23T18:49:41,462 INFO [2eda60943ea1:37117 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T18:49:41,462 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=aafadebc54e468784234072b47bbd24c, regionState=OPENING, regionLocation=2eda60943ea1,43797,1732387765099 2024-11-23T18:49:41,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, REOPEN/MOVE because future has completed 2024-11-23T18:49:41,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,43797,1732387765099}] 2024-11-23T18:49:41,623 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:41,624 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => aafadebc54e468784234072b47bbd24c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:41,624 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:41,624 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:41,624 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:41,624 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:41,626 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:41,627 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aafadebc54e468784234072b47bbd24c columnFamilyName cf1 2024-11-23T18:49:41,627 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:41,635 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/548a1e14ded3450b96f2965ee6e0a2cf 2024-11-23T18:49:41,636 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(327): Store=aafadebc54e468784234072b47bbd24c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:41,636 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:41,637 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aafadebc54e468784234072b47bbd24c columnFamilyName cf2 2024-11-23T18:49:41,637 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:41,638 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(327): Store=aafadebc54e468784234072b47bbd24c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:41,638 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:41,639 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:41,641 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:41,642 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:41,642 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:41,643 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-23T18:49:41,644 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:41,645 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened aafadebc54e468784234072b47bbd24c; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70465962, jitterRate=0.05002465844154358}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-23T18:49:41,645 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:41,646 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for aafadebc54e468784234072b47bbd24c: Running coprocessor pre-open hook at 1732387781624Writing region info on filesystem at 1732387781624Initializing all the Stores at 1732387781626 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387781626Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387781626Cleaning up temporary data from old regions at 1732387781642 (+16 ms)Running coprocessor post-open hooks at 1732387781645 (+3 ms)Region opened successfully at 1732387781646 (+1 ms) 2024-11-23T18:49:41,647 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., pid=9, masterSystemTime=1732387781619 2024-11-23T18:49:41,650 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:41,650 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:41,651 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=aafadebc54e468784234072b47bbd24c, regionState=OPEN, openSeqNum=9, regionLocation=2eda60943ea1,43797,1732387765099 2024-11-23T18:49:41,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,43797,1732387765099 because future has completed 2024-11-23T18:49:41,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-23T18:49:41,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,43797,1732387765099 in 191 msec 2024-11-23T18:49:41,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, REOPEN/MOVE in 995 msec 2024-11-23T18:49:41,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T18:49:41,675 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T18:49:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32913 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:37196 deadline: 1732387841679, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2eda60943ea1 port=43797 startCode=1732387765099. As of locationSeqNum=5. 2024-11-23T18:49:41,701 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,32913,1732387765000, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,32913,1732387765000, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2eda60943ea1 port=43797 startCode=1732387765099. As of locationSeqNum=5. 2024-11-23T18:49:41,702 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,32913,1732387765000, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2eda60943ea1 port=43797 startCode=1732387765099. As of locationSeqNum=5. 2024-11-23T18:49:41,702 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,32913,1732387765000, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,43797,1732387765099, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2eda60943ea1 port=43797 startCode=1732387765099. As of locationSeqNum=5. 2024-11-23T18:49:41,816 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:49:41,818 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:49:41,829 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing aafadebc54e468784234072b47bbd24c 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-23T18:49:41,851 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/.tmp/cf1/bf144b3328254c118bcbf3415c8f6aa6 is 29, key is r1/cf1:/1732387781819/DeleteFamily/seqid=0 2024-11-23T18:49:41,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741888_1066 (size=4906) 2024-11-23T18:49:41,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741888_1066 (size=4906) 2024-11-23T18:49:41,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741888_1066 (size=4906) 2024-11-23T18:49:41,862 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/.tmp/cf1/bf144b3328254c118bcbf3415c8f6aa6 2024-11-23T18:49:41,868 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bf144b3328254c118bcbf3415c8f6aa6 2024-11-23T18:49:41,883 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/.tmp/cf2/54507299eee8401d838e5bb19c0d62a2 is 29, key is r1/cf2:/1732387781819/DeleteFamily/seqid=0 2024-11-23T18:49:41,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741889_1067 (size=4906) 2024-11-23T18:49:41,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741889_1067 (size=4906) 2024-11-23T18:49:41,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741889_1067 (size=4906) 2024-11-23T18:49:41,890 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/.tmp/cf2/54507299eee8401d838e5bb19c0d62a2 2024-11-23T18:49:41,898 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 54507299eee8401d838e5bb19c0d62a2 2024-11-23T18:49:41,899 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/.tmp/cf1/bf144b3328254c118bcbf3415c8f6aa6 as hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/bf144b3328254c118bcbf3415c8f6aa6 2024-11-23T18:49:41,908 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bf144b3328254c118bcbf3415c8f6aa6 2024-11-23T18:49:41,908 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/bf144b3328254c118bcbf3415c8f6aa6, entries=1, sequenceid=12, filesize=4.8 K 2024-11-23T18:49:41,910 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/.tmp/cf2/54507299eee8401d838e5bb19c0d62a2 as hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf2/54507299eee8401d838e5bb19c0d62a2 2024-11-23T18:49:41,917 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 54507299eee8401d838e5bb19c0d62a2 2024-11-23T18:49:41,917 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf2/54507299eee8401d838e5bb19c0d62a2, entries=1, sequenceid=12, filesize=4.8 K 2024-11-23T18:49:41,919 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for aafadebc54e468784234072b47bbd24c in 90ms, sequenceid=12, compaction requested=false 2024-11-23T18:49:41,919 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for aafadebc54e468784234072b47bbd24c: 2024-11-23T18:49:41,921 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-23T18:49:41,923 DEBUG [Time-limited test {}] regionserver.HStore(1541): aafadebc54e468784234072b47bbd24c/cf1 is initiating major compaction (all files) 2024-11-23T18:49:41,923 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T18:49:41,923 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:41,923 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of aafadebc54e468784234072b47bbd24c/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:41,924 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/548a1e14ded3450b96f2965ee6e0a2cf, hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/bf144b3328254c118bcbf3415c8f6aa6] into tmpdir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/.tmp, totalSize=9.5 K 2024-11-23T18:49:41,925 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 548a1e14ded3450b96f2965ee6e0a2cf, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732387780643 2024-11-23T18:49:41,925 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting bf144b3328254c118bcbf3415c8f6aa6, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-23T18:49:41,936 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): aafadebc54e468784234072b47bbd24c#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T18:49:41,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741890_1068 (size=4626) 2024-11-23T18:49:41,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741890_1068 (size=4626) 2024-11-23T18:49:41,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741890_1068 (size=4626) 2024-11-23T18:49:41,957 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/.tmp/cf1/8775858cbb91476aa9024c2ed702a7c4 as hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/8775858cbb91476aa9024c2ed702a7c4 2024-11-23T18:49:41,975 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in aafadebc54e468784234072b47bbd24c/cf1 of aafadebc54e468784234072b47bbd24c into 8775858cbb91476aa9024c2ed702a7c4(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:49:41,975 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for aafadebc54e468784234072b47bbd24c: 2024-11-23T18:49:41,975 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-23T18:49:41,975 DEBUG [Time-limited test {}] regionserver.HStore(1541): aafadebc54e468784234072b47bbd24c/cf2 is initiating major compaction (all files) 2024-11-23T18:49:41,975 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T18:49:41,976 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:49:41,976 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of aafadebc54e468784234072b47bbd24c/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:41,976 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf2/54507299eee8401d838e5bb19c0d62a2] into tmpdir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/.tmp, totalSize=4.8 K 2024-11-23T18:49:41,977 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 54507299eee8401d838e5bb19c0d62a2, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-23T18:49:41,984 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): aafadebc54e468784234072b47bbd24c#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T18:49:41,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741891_1069 (size=4592) 2024-11-23T18:49:41,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741891_1069 (size=4592) 2024-11-23T18:49:41,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741891_1069 (size=4592) 2024-11-23T18:49:42,001 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/.tmp/cf2/391b77986a4748ff805466ba476d5059 as hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf2/391b77986a4748ff805466ba476d5059 2024-11-23T18:49:42,009 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in aafadebc54e468784234072b47bbd24c/cf2 of aafadebc54e468784234072b47bbd24c into 391b77986a4748ff805466ba476d5059(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:49:42,010 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for aafadebc54e468784234072b47bbd24c: 2024-11-23T18:49:42,015 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=aafadebc54e468784234072b47bbd24c, source=2eda60943ea1,43797,1732387765099, destination=2eda60943ea1,32913,1732387765000, warming up region on 2eda60943ea1,32913,1732387765000 2024-11-23T18:49:42,015 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=aafadebc54e468784234072b47bbd24c, source=2eda60943ea1,43797,1732387765099, destination=2eda60943ea1,32913,1732387765000, running balancer 2024-11-23T18:49:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, REOPEN/MOVE 2024-11-23T18:49:42,017 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, REOPEN/MOVE 2024-11-23T18:49:42,019 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=aafadebc54e468784234072b47bbd24c, regionState=CLOSING, regionLocation=2eda60943ea1,43797,1732387765099 2024-11-23T18:49:42,019 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32913 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32913 {}] regionserver.HRegion(7855): Warmup {ENCODED => aafadebc54e468784234072b47bbd24c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32913 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:42,020 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,021 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aafadebc54e468784234072b47bbd24c columnFamilyName cf1 2024-11-23T18:49:42,021 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:42,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, REOPEN/MOVE because future has completed 2024-11-23T18:49:42,022 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T18:49:42,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,43797,1732387765099}] 2024-11-23T18:49:42,030 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/548a1e14ded3450b96f2965ee6e0a2cf 2024-11-23T18:49:42,036 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/8775858cbb91476aa9024c2ed702a7c4 2024-11-23T18:49:42,041 INFO [StoreFileOpener-aafadebc54e468784234072b47bbd24c-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bf144b3328254c118bcbf3415c8f6aa6 2024-11-23T18:49:42,041 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/bf144b3328254c118bcbf3415c8f6aa6 2024-11-23T18:49:42,041 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(327): Store=aafadebc54e468784234072b47bbd24c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:42,042 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,043 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aafadebc54e468784234072b47bbd24c columnFamilyName cf2 2024-11-23T18:49:42,043 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:42,050 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf2/391b77986a4748ff805466ba476d5059 2024-11-23T18:49:42,056 INFO [StoreFileOpener-aafadebc54e468784234072b47bbd24c-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 54507299eee8401d838e5bb19c0d62a2 2024-11-23T18:49:42,057 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf2/54507299eee8401d838e5bb19c0d62a2 2024-11-23T18:49:42,057 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(327): Store=aafadebc54e468784234072b47bbd24c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32913 {}] regionserver.HRegion(1722): Closing aafadebc54e468784234072b47bbd24c, disabling compactions & flushes 2024-11-23T18:49:42,057 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32913 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32913 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32913 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. after waiting 0 ms 2024-11-23T18:49:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32913 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:42,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32913 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32913 {}] regionserver.HRegion(1676): Region close journal for aafadebc54e468784234072b47bbd24c: Waiting for close lock at 1732387782057Disabling compacts and flushes for region at 1732387782057Disabling writes for close at 1732387782057Writing region close event to WAL at 1732387782058 (+1 ms)Closed at 1732387782058 2024-11-23T18:49:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-23T18:49:42,176 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,177 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T18:49:42,177 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing aafadebc54e468784234072b47bbd24c, disabling compactions & flushes 2024-11-23T18:49:42,177 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:42,177 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:42,177 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. after waiting 0 ms 2024-11-23T18:49:42,177 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:42,177 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/548a1e14ded3450b96f2965ee6e0a2cf, hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/bf144b3328254c118bcbf3415c8f6aa6] to archive 2024-11-23T18:49:42,181 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T18:49:42,185 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/548a1e14ded3450b96f2965ee6e0a2cf to hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/548a1e14ded3450b96f2965ee6e0a2cf 2024-11-23T18:49:42,187 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/bf144b3328254c118bcbf3415c8f6aa6 to hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/bf144b3328254c118bcbf3415c8f6aa6 2024-11-23T18:49:42,198 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf2/54507299eee8401d838e5bb19c0d62a2] to archive 2024-11-23T18:49:42,199 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T18:49:42,201 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf2/54507299eee8401d838e5bb19c0d62a2 to hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf2/54507299eee8401d838e5bb19c0d62a2 2024-11-23T18:49:42,206 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-23T18:49:42,206 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:42,206 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for aafadebc54e468784234072b47bbd24c: Waiting for close lock at 1732387782177Running coprocessor pre-close hooks at 1732387782177Disabling compacts and flushes for region at 1732387782177Disabling writes for close at 1732387782177Writing region close event to WAL at 1732387782202 (+25 ms)Running coprocessor post-close hooks at 1732387782206 (+4 ms)Closed at 1732387782206 2024-11-23T18:49:42,206 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding aafadebc54e468784234072b47bbd24c move to 2eda60943ea1,32913,1732387765000 record at close sequenceid=12 2024-11-23T18:49:42,209 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,209 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=aafadebc54e468784234072b47bbd24c, regionState=CLOSED 2024-11-23T18:49:42,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,43797,1732387765099 because future has completed 2024-11-23T18:49:42,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-23T18:49:42,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,43797,1732387765099 in 191 msec 2024-11-23T18:49:42,216 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, REOPEN/MOVE; state=CLOSED, location=2eda60943ea1,32913,1732387765000; forceNewPlan=false, retain=false 2024-11-23T18:49:42,366 INFO [2eda60943ea1:37117 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T18:49:42,367 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=aafadebc54e468784234072b47bbd24c, regionState=OPENING, regionLocation=2eda60943ea1,32913,1732387765000 2024-11-23T18:49:42,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, REOPEN/MOVE because future has completed 2024-11-23T18:49:42,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,32913,1732387765000}] 2024-11-23T18:49:42,534 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:42,535 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => aafadebc54e468784234072b47bbd24c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:42,536 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,536 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:42,536 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,536 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,539 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,540 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aafadebc54e468784234072b47bbd24c columnFamilyName cf1 2024-11-23T18:49:42,540 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:42,547 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/8775858cbb91476aa9024c2ed702a7c4 2024-11-23T18:49:42,548 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(327): Store=aafadebc54e468784234072b47bbd24c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:42,548 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,550 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aafadebc54e468784234072b47bbd24c columnFamilyName cf2 2024-11-23T18:49:42,550 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:42,557 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf2/391b77986a4748ff805466ba476d5059 2024-11-23T18:49:42,557 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(327): Store=aafadebc54e468784234072b47bbd24c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:42,557 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,558 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,560 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,560 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,560 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,561 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-23T18:49:42,563 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,564 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened aafadebc54e468784234072b47bbd24c; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74310997, jitterRate=0.1073201447725296}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-23T18:49:42,564 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,565 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for aafadebc54e468784234072b47bbd24c: Running coprocessor pre-open hook at 1732387782537Writing region info on filesystem at 1732387782537Initializing all the Stores at 1732387782538 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387782538Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387782539 (+1 ms)Cleaning up temporary data from old regions at 1732387782560 (+21 ms)Running coprocessor post-open hooks at 1732387782564 (+4 ms)Region opened successfully at 1732387782565 (+1 ms) 2024-11-23T18:49:42,566 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., pid=12, masterSystemTime=1732387782526 2024-11-23T18:49:42,568 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:42,568 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:42,569 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=aafadebc54e468784234072b47bbd24c, regionState=OPEN, openSeqNum=18, regionLocation=2eda60943ea1,32913,1732387765000 2024-11-23T18:49:42,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,32913,1732387765000 because future has completed 2024-11-23T18:49:42,578 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-23T18:49:42,578 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,32913,1732387765000 in 201 msec 2024-11-23T18:49:42,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, REOPEN/MOVE in 563 msec 2024-11-23T18:49:42,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T18:49:42,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T18:49:42,623 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 2eda60943ea1,32913,1732387765000: testing ***** 2024-11-23T18:49:42,623 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-23T18:49:42,625 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-23T18:49:42,626 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-23T18:49:42,629 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-23T18:49:42,630 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-23T18:49:42,638 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 332861984 }, "NonHeapMemoryUsage": { "committed": 172032000, "init": 7667712, "max": -1, "used": 169355472 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "2eda60943ea1", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2068, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 10, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 9, "ProcessCallTime_95th_percentile": 9, "ProcessCallTime_98th_percentile": 9, "ProcessCallTime_99th_percentile": 9, "ProcessCallTime_99.9th_percentile": 9, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 10, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 9, "TotalCallTime_95th_percentile": 9, "TotalCallTime_98th_percentile": 9, "TotalCallTime_99th_percentile": 9, "TotalCallTime_99.9th_percentile": 9, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "2eda60943ea1", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:57018", "tag.serverName": "2eda60943ea1,43797,1732387765099", "tag.clusterId": "c7282851-006a-445f-a51c-1e6fe6ae5ccd", "tag.Context": "regionserver", "tag.Hostname": "2eda60943ea1", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732387765099, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-23T18:49:42,642 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37117 {}] master.MasterRpcServices(700): 2eda60943ea1,32913,1732387765000 reported a fatal error: ***** ABORTING region server 2eda60943ea1,32913,1732387765000: testing ***** 2024-11-23T18:49:42,644 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2eda60943ea1,32913,1732387765000' ***** 2024-11-23T18:49:42,644 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-23T18:49:42,644 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T18:49:42,644 INFO [RS:0;2eda60943ea1:32913 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-23T18:49:42,645 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T18:49:42,645 INFO [RS:0;2eda60943ea1:32913 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-23T18:49:42,645 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.HRegionServer(3091): Received CLOSE for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,645 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.HRegionServer(956): aborting server 2eda60943ea1,32913,1732387765000 2024-11-23T18:49:42,645 INFO [RS:0;2eda60943ea1:32913 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:49:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43797 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:37912 deadline: 1732387842645, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2eda60943ea1 port=32913 startCode=1732387765000. As of locationSeqNum=12. 2024-11-23T18:49:42,646 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aafadebc54e468784234072b47bbd24c, disabling compactions & flushes 2024-11-23T18:49:42,646 INFO [RS:0;2eda60943ea1:32913 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2eda60943ea1:32913. 2024-11-23T18:49:42,646 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:42,646 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:42,646 DEBUG [RS:0;2eda60943ea1:32913 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:49:42,646 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. after waiting 0 ms 2024-11-23T18:49:42,646 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:42,646 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,43797,1732387765099, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,43797,1732387765099, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2eda60943ea1 port=32913 startCode=1732387765000. As of locationSeqNum=12. 2024-11-23T18:49:42,646 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,43797,1732387765099, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2eda60943ea1 port=32913 startCode=1732387765000. As of locationSeqNum=12. 2024-11-23T18:49:42,646 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,43797,1732387765099, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,32913,1732387765000, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2eda60943ea1 port=32913 startCode=1732387765000. As of locationSeqNum=12. 2024-11-23T18:49:42,646 DEBUG [RS:0;2eda60943ea1:32913 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:49:42,647 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T18:49:42,647 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T18:49:42,647 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T18:49:42,647 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T18:49:42,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2eda60943ea1,32913,1732387765000 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:42,648 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-23T18:49:42,648 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T18:49:42,648 DEBUG [RS:0;2eda60943ea1:32913 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, aafadebc54e468784234072b47bbd24c=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.} 2024-11-23T18:49:42,648 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T18:49:42,648 INFO [regionserver/2eda60943ea1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:49:42,648 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T18:49:42,649 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T18:49:42,649 DEBUG [RS:0;2eda60943ea1:32913 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:42,649 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T18:49:42,649 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2eda60943ea1,32913,1732387765000 aborting 2024-11-23T18:49:42,649 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2eda60943ea1,32913,1732387765000 aborting 2024-11-23T18:49:42,649 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1 from cache 2024-11-23T18:49:42,650 ERROR [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-23T18:49:42,650 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T18:49:42,650 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T18:49:42,650 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:42,650 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732387782648Running coprocessor pre-close hooks at 1732387782648Disabling compacts and flushes for region at 1732387782648Disabling writes for close at 1732387782649 (+1 ms)Writing region close event to WAL at 1732387782650 (+1 ms)Running coprocessor post-close hooks at 1732387782650Closed at 1732387782650 2024-11-23T18:49:42,650 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aafadebc54e468784234072b47bbd24c: Waiting for close lock at 1732387782645Running coprocessor pre-close hooks at 1732387782646 (+1 ms)Disabling compacts and flushes for region at 1732387782646Disabling writes for close at 1732387782646Writing region close event to WAL at 1732387782650 (+4 ms)Running coprocessor post-close hooks at 1732387782650Closed at 1732387782650 2024-11-23T18:49:42,650 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T18:49:42,650 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:42,755 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:49:42,758 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1] 2024-11-23T18:49:42,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2eda60943ea1,32913,1732387765000 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:42,760 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2eda60943ea1,32913,1732387765000 aborting 2024-11-23T18:49:42,760 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2eda60943ea1,32913,1732387765000 aborting 2024-11-23T18:49:42,761 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1 from cache 2024-11-23T18:49:42,849 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.HRegionServer(976): stopping server 2eda60943ea1,32913,1732387765000; all regions closed. 2024-11-23T18:49:42,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741836_1012 (size=3561) 2024-11-23T18:49:42,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741836_1012 (size=3561) 2024-11-23T18:49:42,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741836_1012 (size=3561) 2024-11-23T18:49:42,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741833_1009 (size=1407) 2024-11-23T18:49:42,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741833_1009 (size=1407) 2024-11-23T18:49:42,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741833_1009 (size=1407) 2024-11-23T18:49:42,861 DEBUG [RS:0;2eda60943ea1:32913 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:49:42,861 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:49:42,861 INFO [RS:0;2eda60943ea1:32913 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:49:42,861 INFO [RS:0;2eda60943ea1:32913 {}] hbase.ChoreService(370): Chore service for: regionserver/2eda60943ea1:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T18:49:42,862 INFO [RS:0;2eda60943ea1:32913 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:49:42,862 INFO [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:49:42,862 INFO [RS:0;2eda60943ea1:32913 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32913 2024-11-23T18:49:42,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32913-0x101690770ba0001, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2eda60943ea1,32913,1732387765000 2024-11-23T18:49:42,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:49:42,929 INFO [RS:0;2eda60943ea1:32913 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:49:42,943 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2eda60943ea1,32913,1732387765000] 2024-11-23T18:49:42,953 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2eda60943ea1,32913,1732387765000 already deleted, retry=false 2024-11-23T18:49:42,953 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 2eda60943ea1,32913,1732387765000 on 2eda60943ea1,37117,1732387764258 2024-11-23T18:49:42,962 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 2eda60943ea1,32913,1732387765000, splitWal=true, meta=true 2024-11-23T18:49:42,966 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:49:42,967 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 2eda60943ea1,32913,1732387765000 (carryingMeta=true) 2eda60943ea1,32913,1732387765000/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@54784ad9[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-23T18:49:42,967 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 2eda60943ea1,32913,1732387765000, splitWal=true, meta=true 2024-11-23T18:49:42,967 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1] 2024-11-23T18:49:42,968 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 2eda60943ea1,32913,1732387765000, splitWal=true, meta=true, isMeta: true 2024-11-23T18:49:42,969 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 2eda60943ea1:32913 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 2eda60943ea1/172.17.0.2:32913 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:42,969 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1, error=java.net.ConnectException: Call to address=2eda60943ea1:32913 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 2eda60943ea1/172.17.0.2:32913 2024-11-23T18:49:42,969 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-23T18:49:42,970 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1 from cache 2024-11-23T18:49:42,970 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 2eda60943ea1:32913 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 2eda60943ea1/172.17.0.2:32913 2024-11-23T18:49:42,971 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting 2024-11-23T18:49:42,972 INFO [PEWorker-2 {}] master.SplitWALManager(105): 2eda60943ea1,32913,1732387765000 WAL count=1, meta=true 2024-11-23T18:49:42,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta}] 2024-11-23T18:49:42,981 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=2eda60943ea1,43797,1732387765099 2024-11-23T18:49:42,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta, worker=2eda60943ea1,43797,1732387765099}] 2024-11-23T18:49:43,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32913-0x101690770ba0001, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:49:43,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32913-0x101690770ba0001, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:49:43,043 INFO [RS:0;2eda60943ea1:32913 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:49:43,044 INFO [RS:0;2eda60943ea1:32913 {}] regionserver.HRegionServer(1031): Exiting; stopping=2eda60943ea1,32913,1732387765000; zookeeper connection closed. 2024-11-23T18:49:43,044 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@cafa493 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@cafa493 2024-11-23T18:49:43,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-23T18:49:43,162 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta, size=3.5 K (3561bytes) 2024-11-23T18:49:43,162 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta 2024-11-23T18:49:43,163 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta after 1ms 2024-11-23T18:49:43,165 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:43,165 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta took 4ms 2024-11-23T18:49:43,173 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-23T18:49:43,174 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta so closing down 2024-11-23T18:49:43,174 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:49:43,175 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta.temp 2024-11-23T18:49:43,176 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta.temp 2024-11-23T18:49:43,177 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:49:43,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741892_1070 (size=3346) 2024-11-23T18:49:43,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741892_1070 (size=3346) 2024-11-23T18:49:43,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741892_1070 (size=3346) 2024-11-23T18:49:43,185 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-23T18:49:43,187 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta.temp to hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-23T18:49:43,187 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 21 ms; skipped=1; WAL=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta, size=3.5 K, length=3561, corrupted=false, cancelled=false 2024-11-23T18:49:43,187 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta, journal: Splitting hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta, size=3.5 K (3561bytes) at 1732387783162Finishing writing output for hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta so closing down at 1732387783174 (+12 ms)Creating recovered edits writer path=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta.temp at 1732387783176 (+2 ms)3 split writer threads finished at 1732387783177 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732387783185 (+8 ms)Rename recovered edits hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta.temp to hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1732387783187 (+2 ms)Processed 16 edits across 1 Regions in 21 ms; skipped=1; WAL=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta, size=3.5 K, length=3561, corrupted=false, cancelled=false at 1732387783187 2024-11-23T18:49:43,187 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta 2024-11-23T18:49:43,189 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-23T18:49:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-23T18:49:43,196 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta to hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/oldWALs 2024-11-23T18:49:43,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-23T18:49:43,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta, worker=2eda60943ea1,43797,1732387765099 in 213 msec 2024-11-23T18:49:43,201 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=2eda60943ea1,43797,1732387765099 2024-11-23T18:49:43,205 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-23T18:49:43,205 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 2eda60943ea1%2C32913%2C1732387765000.meta.1732387767123.meta, worker=2eda60943ea1,43797,1732387765099 in 228 msec 2024-11-23T18:49:43,207 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting dir is empty, no logs to split. 2024-11-23T18:49:43,207 INFO [PEWorker-2 {}] master.SplitWALManager(105): 2eda60943ea1,32913,1732387765000 WAL count=0, meta=true 2024-11-23T18:49:43,207 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 2eda60943ea1,32913,1732387765000 WAL splitting is done? wals=0, meta=true 2024-11-23T18:49:43,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T18:49:43,212 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:49:43,213 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-23T18:49:43,276 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:49:43,278 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1] 2024-11-23T18:49:43,278 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 2eda60943ea1:32913 this server is in the failed servers list 2024-11-23T18:49:43,279 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=2eda60943ea1:32913 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2eda60943ea1:32913 2024-11-23T18:49:43,279 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2eda60943ea1:32913 2024-11-23T18:49:43,279 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1 from cache 2024-11-23T18:49:43,364 DEBUG [2eda60943ea1:37117 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-23T18:49:43,364 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(204): Hosts are {2eda60943ea1=0} racks are {/default-rack=0} 2024-11-23T18:49:43,364 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T18:49:43,364 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T18:49:43,364 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T18:49:43,364 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T18:49:43,364 INFO [2eda60943ea1:37117 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T18:49:43,364 INFO [2eda60943ea1:37117 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T18:49:43,364 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T18:49:43,365 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2eda60943ea1,43797,1732387765099 2024-11-23T18:49:43,366 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,43797,1732387765099, state=OPENING 2024-11-23T18:49:43,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:49:43,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:49:43,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:49:43,385 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:49:43,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:49:43,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:49:43,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:49:43,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,43797,1732387765099}] 2024-11-23T18:49:43,542 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T18:49:43,542 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:43,542 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T18:49:43,544 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C43797%2C1732387765099.meta, suffix=.meta, logDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,43797,1732387765099, archiveDir=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/oldWALs, maxLogs=32 2024-11-23T18:49:43,556 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,43797,1732387765099/2eda60943ea1%2C43797%2C1732387765099.meta.1732387783544.meta, exclude list is [], retry=0 2024-11-23T18:49:43,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:43,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:43,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:43,561 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,43797,1732387765099/2eda60943ea1%2C43797%2C1732387765099.meta.1732387783544.meta 2024-11-23T18:49:43,562 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:43381:43381)] 2024-11-23T18:49:43,562 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:43,562 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T18:49:43,562 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T18:49:43,562 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T18:49:43,562 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T18:49:43,562 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:43,562 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T18:49:43,563 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T18:49:43,564 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T18:49:43,565 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T18:49:43,565 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:43,565 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:49:43,565 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T18:49:43,566 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T18:49:43,566 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:43,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:49:43,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T18:49:43,568 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T18:49:43,568 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:43,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:49:43,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T18:49:43,569 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T18:49:43,569 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:43,570 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:49:43,570 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T18:49:43,570 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740 2024-11-23T18:49:43,572 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740 2024-11-23T18:49:43,572 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-23T18:49:43,575 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000018: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:43,577 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-23T18:49:43,578 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-23T18:49:43,600 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/.tmp/info/8d3129a2b0f44e0da78e09b128dfd0bb is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c./info:regioninfo/1732387782569/Put/seqid=0 2024-11-23T18:49:43,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741894_1072 (size=11177) 2024-11-23T18:49:43,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741894_1072 (size=11177) 2024-11-23T18:49:43,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741894_1072 (size=11177) 2024-11-23T18:49:43,608 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/.tmp/info/8d3129a2b0f44e0da78e09b128dfd0bb 2024-11-23T18:49:43,629 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/.tmp/ns/6e3cf07ed4364edd8f2fb3c377c2a587 is 43, key is default/ns:d/1732387767287/Put/seqid=0 2024-11-23T18:49:43,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741895_1073 (size=5153) 2024-11-23T18:49:43,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741895_1073 (size=5153) 2024-11-23T18:49:43,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741895_1073 (size=5153) 2024-11-23T18:49:43,638 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/.tmp/ns/6e3cf07ed4364edd8f2fb3c377c2a587 2024-11-23T18:49:43,657 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/.tmp/table/7ec04ea33f3549f58029f9bba503d9f0 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732387780393/Put/seqid=0 2024-11-23T18:49:43,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741896_1074 (size=5431) 2024-11-23T18:49:43,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741896_1074 (size=5431) 2024-11-23T18:49:43,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741896_1074 (size=5431) 2024-11-23T18:49:43,663 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/.tmp/table/7ec04ea33f3549f58029f9bba503d9f0 2024-11-23T18:49:43,669 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/.tmp/info/8d3129a2b0f44e0da78e09b128dfd0bb as hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/info/8d3129a2b0f44e0da78e09b128dfd0bb 2024-11-23T18:49:43,675 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/info/8d3129a2b0f44e0da78e09b128dfd0bb, entries=36, sequenceid=18, filesize=10.9 K 2024-11-23T18:49:43,676 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/.tmp/ns/6e3cf07ed4364edd8f2fb3c377c2a587 as hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/ns/6e3cf07ed4364edd8f2fb3c377c2a587 2024-11-23T18:49:43,683 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/ns/6e3cf07ed4364edd8f2fb3c377c2a587, entries=2, sequenceid=18, filesize=5.0 K 2024-11-23T18:49:43,684 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/.tmp/table/7ec04ea33f3549f58029f9bba503d9f0 as hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/table/7ec04ea33f3549f58029f9bba503d9f0 2024-11-23T18:49:43,690 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/table/7ec04ea33f3549f58029f9bba503d9f0, entries=2, sequenceid=18, filesize=5.3 K 2024-11-23T18:49:43,690 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 113ms, sequenceid=18, compaction requested=false; wal=null 2024-11-23T18:49:43,691 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-23T18:49:43,692 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T18:49:43,692 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T18:49:43,693 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T18:49:43,695 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T18:49:43,697 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-23T18:49:43,698 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73263753, jitterRate=0.09171499311923981}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T18:49:43,698 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T18:49:43,699 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732387783563Writing region info on filesystem at 1732387783563Initializing all the Stores at 1732387783563Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387783564 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387783564Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387783564Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387783564Obtaining lock to block concurrent updates at 1732387783578 (+14 ms)Preparing flush snapshotting stores in 1588230740 at 1732387783578Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1732387783578Flushing stores of hbase:meta,,1.1588230740 at 1732387783578Flushing 1588230740/info: creating writer at 1732387783578Flushing 1588230740/info: appending metadata at 1732387783600 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732387783600Flushing 1588230740/ns: creating writer at 1732387783614 (+14 ms)Flushing 1588230740/ns: appending metadata at 1732387783628 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732387783628Flushing 1588230740/table: creating writer at 1732387783643 (+15 ms)Flushing 1588230740/table: appending metadata at 1732387783656 (+13 ms)Flushing 1588230740/table: closing flushed file at 1732387783656Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b9cb0ee: reopening flushed file at 1732387783668 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16902481: reopening flushed file at 1732387783675 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@285abcc8: reopening flushed file at 1732387783683 (+8 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 113ms, sequenceid=18, compaction requested=false; wal=null at 1732387783690 (+7 ms)Cleaning up temporary data from old regions at 1732387783692 (+2 ms)Running coprocessor post-open hooks at 1732387783698 (+6 ms)Region opened successfully at 1732387783699 (+1 ms) 2024-11-23T18:49:43,700 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1732387783538 2024-11-23T18:49:43,702 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T18:49:43,702 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T18:49:43,703 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=2eda60943ea1,43797,1732387765099 2024-11-23T18:49:43,704 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,43797,1732387765099, state=OPEN 2024-11-23T18:49:43,705 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37117 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2eda60943ea1,43797,1732387765099, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-23T18:49:43,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:49:43,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:49:43,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:49:43,711 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:49:43,711 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:49:43,711 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:49:43,711 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2eda60943ea1,43797,1732387765099 2024-11-23T18:49:43,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-23T18:49:43,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,43797,1732387765099 in 326 msec 2024-11-23T18:49:43,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-23T18:49:43,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 505 msec 2024-11-23T18:49:43,718 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 2eda60943ea1,32913,1732387765000 had 2 regions 2024-11-23T18:49:43,720 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 2eda60943ea1,32913,1732387765000, splitWal=true, meta=true, isMeta: false 2024-11-23T18:49:43,722 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2eda60943ea1,32913,1732387765000 WAL count=1, meta=false 2024-11-23T18:49:43,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 2eda60943ea1%2C32913%2C1732387765000.1732387766690}] 2024-11-23T18:49:43,725 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=2eda60943ea1,43797,1732387765099 2024-11-23T18:49:43,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 2eda60943ea1%2C32913%2C1732387765000.1732387766690, worker=2eda60943ea1,43797,1732387765099}] 2024-11-23T18:49:43,786 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:49:43,787 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,43797,1732387765099, seqNum=-1] 2024-11-23T18:49:43,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-23T18:49:43,895 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.1732387766690, size=1.4 K (1407bytes) 2024-11-23T18:49:43,895 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.1732387766690 2024-11-23T18:49:43,896 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.1732387766690 after 1ms 2024-11-23T18:49:43,898 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.1732387766690: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:43,898 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.1732387766690 took 3ms 2024-11-23T18:49:43,901 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for aafadebc54e468784234072b47bbd24c: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-23T18:49:43,901 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.1732387766690 so closing down 2024-11-23T18:49:43,901 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:49:43,902 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:49:43,902 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.1732387766690, size=1.4 K, length=1407, corrupted=false, cancelled=false 2024-11-23T18:49:43,902 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.1732387766690, journal: Splitting hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.1732387766690, size=1.4 K (1407bytes) at 1732387783895Finishing writing output for hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.1732387766690 so closing down at 1732387783901 (+6 ms)3 split writer threads finished at 1732387783902 (+1 ms)Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.1732387766690, size=1.4 K, length=1407, corrupted=false, cancelled=false at 1732387783902 2024-11-23T18:49:43,902 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.1732387766690 2024-11-23T18:49:43,902 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-23T18:49:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37117 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-23T18:49:43,906 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting/2eda60943ea1%2C32913%2C1732387765000.1732387766690 to hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/oldWALs 2024-11-23T18:49:43,906 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 2eda60943ea1:32913 this server is in the failed servers list 2024-11-23T18:49:43,906 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,32913,1732387765000, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,32913,1732387765000, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=2eda60943ea1:32913 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2eda60943ea1:32913 2024-11-23T18:49:43,907 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,32913,1732387765000, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2eda60943ea1:32913 2024-11-23T18:49:43,907 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,32913,1732387765000, seqNum=12 from cache 2024-11-23T18:49:43,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-23T18:49:43,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2eda60943ea1%2C32913%2C1732387765000.1732387766690, worker=2eda60943ea1,43797,1732387765099 in 180 msec 2024-11-23T18:49:43,911 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=2eda60943ea1,43797,1732387765099 2024-11-23T18:49:43,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-23T18:49:43,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 2eda60943ea1%2C32913%2C1732387765000.1732387766690, worker=2eda60943ea1,43797,1732387765099 in 189 msec 2024-11-23T18:49:43,916 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/WALs/2eda60943ea1,32913,1732387765000-splitting dir is empty, no logs to split. 2024-11-23T18:49:43,916 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2eda60943ea1,32913,1732387765000 WAL count=0, meta=false 2024-11-23T18:49:43,916 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 2eda60943ea1,32913,1732387765000 WAL splitting is done? wals=0, meta=false 2024-11-23T18:49:43,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, ASSIGN}] 2024-11-23T18:49:43,919 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, ASSIGN 2024-11-23T18:49:43,921 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-23T18:49:44,071 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(204): Hosts are {2eda60943ea1=0} racks are {/default-rack=0} 2024-11-23T18:49:44,072 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T18:49:44,072 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T18:49:44,072 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T18:49:44,072 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T18:49:44,072 INFO [2eda60943ea1:37117 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T18:49:44,073 INFO [2eda60943ea1:37117 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T18:49:44,073 DEBUG [2eda60943ea1:37117 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T18:49:44,074 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=aafadebc54e468784234072b47bbd24c, regionState=OPENING, regionLocation=2eda60943ea1,43797,1732387765099 2024-11-23T18:49:44,076 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 2eda60943ea1:32913 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2eda60943ea1/172.17.0.2:32913 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:44,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1, error=java.net.ConnectException: Call to address=2eda60943ea1:32913 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2eda60943ea1/172.17.0.2:32913 2024-11-23T18:49:44,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-23T18:49:44,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,32913,1732387765000, seqNum=-1 from cache 2024-11-23T18:49:44,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 2eda60943ea1:32913 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2eda60943ea1/172.17.0.2:32913 2024-11-23T18:49:44,127 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,32913,1732387765000, seqNum=18] 2024-11-23T18:49:44,127 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 2eda60943ea1:32913 this server is in the failed servers list 2024-11-23T18:49:44,128 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,32913,1732387765000, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,32913,1732387765000, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=2eda60943ea1:32913 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2eda60943ea1:32913 2024-11-23T18:49:44,128 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,32913,1732387765000, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2eda60943ea1:32913 2024-11-23T18:49:44,128 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,32913,1732387765000, seqNum=18 from cache 2024-11-23T18:49:44,185 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:49:44,186 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,43797,1732387765099, seqNum=-1] 2024-11-23T18:49:44,186 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:49:44,188 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47533, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:49:44,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, ASSIGN because future has completed 2024-11-23T18:49:44,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,43797,1732387765099}] 2024-11-23T18:49:44,350 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:44,351 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => aafadebc54e468784234072b47bbd24c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:44,351 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:44,351 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:44,351 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:44,351 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:44,353 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:44,355 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aafadebc54e468784234072b47bbd24c columnFamilyName cf1 2024-11-23T18:49:44,355 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:44,363 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf1/8775858cbb91476aa9024c2ed702a7c4 2024-11-23T18:49:44,363 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(327): Store=aafadebc54e468784234072b47bbd24c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:44,363 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:44,365 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aafadebc54e468784234072b47bbd24c columnFamilyName cf2 2024-11-23T18:49:44,365 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:44,371 DEBUG [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/cf2/391b77986a4748ff805466ba476d5059 2024-11-23T18:49:44,371 INFO [StoreOpener-aafadebc54e468784234072b47bbd24c-1 {}] regionserver.HStore(327): Store=aafadebc54e468784234072b47bbd24c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:44,371 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:44,372 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:44,373 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:44,374 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:44,374 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:44,375 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-23T18:49:44,376 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:44,377 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened aafadebc54e468784234072b47bbd24c; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59293272, jitterRate=-0.11646139621734619}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-23T18:49:44,377 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aafadebc54e468784234072b47bbd24c 2024-11-23T18:49:44,378 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for aafadebc54e468784234072b47bbd24c: Running coprocessor pre-open hook at 1732387784352Writing region info on filesystem at 1732387784352Initializing all the Stores at 1732387784353 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387784353Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387784353Cleaning up temporary data from old regions at 1732387784374 (+21 ms)Running coprocessor post-open hooks at 1732387784377 (+3 ms)Region opened successfully at 1732387784378 (+1 ms) 2024-11-23T18:49:44,379 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., pid=21, masterSystemTime=1732387784346 2024-11-23T18:49:44,381 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:44,381 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:49:44,381 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=aafadebc54e468784234072b47bbd24c, regionState=OPEN, openSeqNum=18, regionLocation=2eda60943ea1,43797,1732387765099 2024-11-23T18:49:44,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,43797,1732387765099 because future has completed 2024-11-23T18:49:44,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-23T18:49:44,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure aafadebc54e468784234072b47bbd24c, server=2eda60943ea1,43797,1732387765099 in 191 msec 2024-11-23T18:49:44,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-23T18:49:44,390 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server 2eda60943ea1,32913,1732387765000 after splitting done 2024-11-23T18:49:44,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=aafadebc54e468784234072b47bbd24c, ASSIGN in 469 msec 2024-11-23T18:49:44,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 2eda60943ea1,32913,1732387765000, splitWal=true, meta=true in 1.4350 sec 2024-11-23T18:49:44,444 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c., hostname=2eda60943ea1,43797,1732387765099, seqNum=18] 2024-11-23T18:49:44,460 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=406 (was 412), OpenFileDescriptor=1042 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=259 (was 282), ProcessCount=11 (was 11), AvailableMemoryMB=8626 (was 8695) 2024-11-23T18:49:44,462 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1042 is superior to 1024 2024-11-23T18:49:44,473 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=406, OpenFileDescriptor=1042, MaxFileDescriptor=1048576, SystemLoadAverage=259, ProcessCount=11, AvailableMemoryMB=8624 2024-11-23T18:49:44,473 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1042 is superior to 1024 2024-11-23T18:49:44,488 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:44,490 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:44,491 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:49:44,493 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-60887473, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/hregion-60887473, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:44,506 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-60887473/hregion-60887473.1732387784494, exclude list is [], retry=0 2024-11-23T18:49:44,509 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:44,510 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:44,510 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:44,512 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-60887473/hregion-60887473.1732387784494 2024-11-23T18:49:44,512 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:43381:43381)] 2024-11-23T18:49:44,512 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c1c0d6f14011bf42266ffbdefd9e1ffe, NAME => 'testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43835/hbase 2024-11-23T18:49:44,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741898_1076 (size=67) 2024-11-23T18:49:44,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741898_1076 (size=67) 2024-11-23T18:49:44,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741898_1076 (size=67) 2024-11-23T18:49:44,521 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:44,522 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,524 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1c0d6f14011bf42266ffbdefd9e1ffe columnFamilyName a 2024-11-23T18:49:44,524 DEBUG [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:44,524 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(327): Store=c1c0d6f14011bf42266ffbdefd9e1ffe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:44,524 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,525 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1c0d6f14011bf42266ffbdefd9e1ffe columnFamilyName b 2024-11-23T18:49:44,525 DEBUG [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:44,526 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(327): Store=c1c0d6f14011bf42266ffbdefd9e1ffe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:44,526 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,527 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1c0d6f14011bf42266ffbdefd9e1ffe columnFamilyName c 2024-11-23T18:49:44,527 DEBUG [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:44,528 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(327): Store=c1c0d6f14011bf42266ffbdefd9e1ffe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:44,528 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,529 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,529 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,530 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,530 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,531 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:49:44,532 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,534 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:49:44,535 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c1c0d6f14011bf42266ffbdefd9e1ffe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70635631, jitterRate=0.05255292356014252}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:49:44,535 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c1c0d6f14011bf42266ffbdefd9e1ffe: Writing region info on filesystem at 1732387784521Initializing all the Stores at 1732387784522 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387784522Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387784522Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387784522Cleaning up temporary data from old regions at 1732387784530 (+8 ms)Region opened successfully at 1732387784535 (+5 ms) 2024-11-23T18:49:44,535 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c1c0d6f14011bf42266ffbdefd9e1ffe, disabling compactions & flushes 2024-11-23T18:49:44,536 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe. 2024-11-23T18:49:44,536 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe. 2024-11-23T18:49:44,536 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe. after waiting 0 ms 2024-11-23T18:49:44,536 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe. 2024-11-23T18:49:44,536 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe. 2024-11-23T18:49:44,536 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c1c0d6f14011bf42266ffbdefd9e1ffe: Waiting for close lock at 1732387784535Disabling compacts and flushes for region at 1732387784535Disabling writes for close at 1732387784536 (+1 ms)Writing region close event to WAL at 1732387784536Closed at 1732387784536 2024-11-23T18:49:44,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741897_1075 (size=95) 2024-11-23T18:49:44,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741897_1075 (size=95) 2024-11-23T18:49:44,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741897_1075 (size=95) 2024-11-23T18:49:44,541 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:49:44,541 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-60887473:(num 1732387784494) 2024-11-23T18:49:44,541 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:49:44,543 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:44,555 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784543, exclude list is [], retry=0 2024-11-23T18:49:44,558 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:44,558 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:44,559 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:44,560 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784543 2024-11-23T18:49:44,560 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:37451:37451)] 2024-11-23T18:49:44,561 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c1c0d6f14011bf42266ffbdefd9e1ffe, NAME => 'testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:44,561 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:44,561 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,561 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,563 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,563 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1c0d6f14011bf42266ffbdefd9e1ffe columnFamilyName a 2024-11-23T18:49:44,564 DEBUG [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:44,564 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(327): Store=c1c0d6f14011bf42266ffbdefd9e1ffe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:44,564 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,565 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1c0d6f14011bf42266ffbdefd9e1ffe columnFamilyName b 2024-11-23T18:49:44,565 DEBUG [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:44,565 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(327): Store=c1c0d6f14011bf42266ffbdefd9e1ffe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:44,565 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,566 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1c0d6f14011bf42266ffbdefd9e1ffe columnFamilyName c 2024-11-23T18:49:44,566 DEBUG [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:44,567 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(327): Store=c1c0d6f14011bf42266ffbdefd9e1ffe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:44,567 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,568 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,569 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,570 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,570 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,570 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:49:44,572 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,572 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c1c0d6f14011bf42266ffbdefd9e1ffe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66112372, jitterRate=-0.014848887920379639}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:49:44,573 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c1c0d6f14011bf42266ffbdefd9e1ffe: Writing region info on filesystem at 1732387784561Initializing all the Stores at 1732387784562 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387784562Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387784562Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387784562Cleaning up temporary data from old regions at 1732387784570 (+8 ms)Region opened successfully at 1732387784573 (+3 ms) 2024-11-23T18:49:44,598 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-23T18:49:44,598 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-23T18:49:44,598 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c1c0d6f14011bf42266ffbdefd9e1ffe 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-23T18:49:44,599 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-23T18:49:44,599 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-23T18:49:44,600 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-23T18:49:44,600 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-23T18:49:44,601 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-23T18:49:44,601 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-23T18:49:44,615 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/.tmp/a/b4d031efbcbe473f9b673fc85bd38538 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732387784573/Put/seqid=0 2024-11-23T18:49:44,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741900_1078 (size=5958) 2024-11-23T18:49:44,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741900_1078 (size=5958) 2024-11-23T18:49:44,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741900_1078 (size=5958) 2024-11-23T18:49:44,623 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/.tmp/a/b4d031efbcbe473f9b673fc85bd38538 2024-11-23T18:49:44,643 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/.tmp/b/8583a55d33f741c9bb7f89750954797c is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732387784582/Put/seqid=0 2024-11-23T18:49:44,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741901_1079 (size=5958) 2024-11-23T18:49:44,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741901_1079 (size=5958) 2024-11-23T18:49:44,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741901_1079 (size=5958) 2024-11-23T18:49:44,649 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/.tmp/b/8583a55d33f741c9bb7f89750954797c 2024-11-23T18:49:44,668 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/.tmp/c/260dc4a3604d44e3b9af92a1d05dc33e is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732387784590/Put/seqid=0 2024-11-23T18:49:44,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741902_1080 (size=5958) 2024-11-23T18:49:44,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741902_1080 (size=5958) 2024-11-23T18:49:44,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741902_1080 (size=5958) 2024-11-23T18:49:44,677 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/.tmp/c/260dc4a3604d44e3b9af92a1d05dc33e 2024-11-23T18:49:44,684 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/.tmp/a/b4d031efbcbe473f9b673fc85bd38538 as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/a/b4d031efbcbe473f9b673fc85bd38538 2024-11-23T18:49:44,690 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/a/b4d031efbcbe473f9b673fc85bd38538, entries=10, sequenceid=33, filesize=5.8 K 2024-11-23T18:49:44,691 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/.tmp/b/8583a55d33f741c9bb7f89750954797c as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/b/8583a55d33f741c9bb7f89750954797c 2024-11-23T18:49:44,697 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/b/8583a55d33f741c9bb7f89750954797c, entries=10, sequenceid=33, filesize=5.8 K 2024-11-23T18:49:44,698 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/.tmp/c/260dc4a3604d44e3b9af92a1d05dc33e as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/c/260dc4a3604d44e3b9af92a1d05dc33e 2024-11-23T18:49:44,704 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/c/260dc4a3604d44e3b9af92a1d05dc33e, entries=10, sequenceid=33, filesize=5.8 K 2024-11-23T18:49:44,705 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for c1c0d6f14011bf42266ffbdefd9e1ffe in 107ms, sequenceid=33, compaction requested=false 2024-11-23T18:49:44,705 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c1c0d6f14011bf42266ffbdefd9e1ffe: 2024-11-23T18:49:44,705 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c1c0d6f14011bf42266ffbdefd9e1ffe, disabling compactions & flushes 2024-11-23T18:49:44,705 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe. 2024-11-23T18:49:44,706 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe. 2024-11-23T18:49:44,706 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe. after waiting 0 ms 2024-11-23T18:49:44,706 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe. 2024-11-23T18:49:44,707 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe. 2024-11-23T18:49:44,707 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c1c0d6f14011bf42266ffbdefd9e1ffe: Waiting for close lock at 1732387784705Disabling compacts and flushes for region at 1732387784705Disabling writes for close at 1732387784706 (+1 ms)Writing region close event to WAL at 1732387784707 (+1 ms)Closed at 1732387784707 2024-11-23T18:49:44,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741899_1077 (size=3385) 2024-11-23T18:49:44,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741899_1077 (size=3385) 2024-11-23T18:49:44,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741899_1077 (size=3385) 2024-11-23T18:49:44,714 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/b/8583a55d33f741c9bb7f89750954797c to hdfs://localhost:43835/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/b/8583a55d33f741c9bb7f89750954797c 2024-11-23T18:49:44,731 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43835/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784543, size=3.3 K (3385bytes) 2024-11-23T18:49:44,731 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784543 2024-11-23T18:49:44,731 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43835/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784543 after 0ms 2024-11-23T18:49:44,734 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784543: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:44,734 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43835/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784543 took 3ms 2024-11-23T18:49:44,736 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43835/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784543 so closing down 2024-11-23T18:49:44,737 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:49:44,737 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732387784543.temp 2024-11-23T18:49:44,738 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/recovered.edits/0000000000000000003-wal.1732387784543.temp 2024-11-23T18:49:44,739 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:49:44,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741903_1081 (size=2944) 2024-11-23T18:49:44,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741903_1081 (size=2944) 2024-11-23T18:49:44,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741903_1081 (size=2944) 2024-11-23T18:49:44,752 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/recovered.edits/0000000000000000003-wal.1732387784543.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-23T18:49:44,753 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/recovered.edits/0000000000000000003-wal.1732387784543.temp to hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/recovered.edits/0000000000000000032 2024-11-23T18:49:44,754 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:43835/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784543, size=3.3 K, length=3385, corrupted=false, cancelled=false 2024-11-23T18:49:44,754 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43835/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784543, journal: Splitting hdfs://localhost:43835/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784543, size=3.3 K (3385bytes) at 1732387784731Finishing writing output for hdfs://localhost:43835/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784543 so closing down at 1732387784736 (+5 ms)Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/recovered.edits/0000000000000000003-wal.1732387784543.temp at 1732387784738 (+2 ms)3 split writer threads finished at 1732387784739 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/recovered.edits/0000000000000000003-wal.1732387784543.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732387784752 (+13 ms)Rename recovered edits hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/recovered.edits/0000000000000000003-wal.1732387784543.temp to hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/recovered.edits/0000000000000000032 at 1732387784753 (+1 ms)Processed 32 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:43835/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784543, size=3.3 K, length=3385, corrupted=false, cancelled=false at 1732387784754 (+1 ms) 2024-11-23T18:49:44,755 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43835/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784543 to hdfs://localhost:43835/hbase/oldWALs/wal.1732387784543 2024-11-23T18:49:44,756 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/recovered.edits/0000000000000000032 2024-11-23T18:49:44,757 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:49:44,758 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:44,760 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T18:49:44,772 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784759, exclude list is [], retry=0 2024-11-23T18:49:44,776 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:44,776 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:44,776 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:44,778 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784759 2024-11-23T18:49:44,778 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:39619:39619)] 2024-11-23T18:49:44,778 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c1c0d6f14011bf42266ffbdefd9e1ffe, NAME => 'testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:44,779 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:44,779 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,779 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,780 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,781 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1c0d6f14011bf42266ffbdefd9e1ffe columnFamilyName a 2024-11-23T18:49:44,781 DEBUG [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:44,788 DEBUG [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/a/b4d031efbcbe473f9b673fc85bd38538 2024-11-23T18:49:44,788 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(327): Store=c1c0d6f14011bf42266ffbdefd9e1ffe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:44,789 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,790 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1c0d6f14011bf42266ffbdefd9e1ffe columnFamilyName b 2024-11-23T18:49:44,790 DEBUG [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:44,791 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(327): Store=c1c0d6f14011bf42266ffbdefd9e1ffe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:44,791 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,792 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1c0d6f14011bf42266ffbdefd9e1ffe columnFamilyName c 2024-11-23T18:49:44,792 DEBUG [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:44,800 DEBUG [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/c/260dc4a3604d44e3b9af92a1d05dc33e 2024-11-23T18:49:44,800 INFO [StoreOpener-c1c0d6f14011bf42266ffbdefd9e1ffe-1 {}] regionserver.HStore(327): Store=c1c0d6f14011bf42266ffbdefd9e1ffe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:44,801 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,801 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,803 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,804 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/recovered.edits/0000000000000000032 2024-11-23T18:49:44,807 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:44,808 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/recovered.edits/0000000000000000032 2024-11-23T18:49:44,808 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c1c0d6f14011bf42266ffbdefd9e1ffe 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-23T18:49:44,824 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/.tmp/b/f9056c54b75244a3b4780083462de0f9 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732387784582/Put/seqid=0 2024-11-23T18:49:44,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741905_1083 (size=5958) 2024-11-23T18:49:44,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741905_1083 (size=5958) 2024-11-23T18:49:44,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741905_1083 (size=5958) 2024-11-23T18:49:44,834 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/.tmp/b/f9056c54b75244a3b4780083462de0f9 2024-11-23T18:49:44,840 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/.tmp/b/f9056c54b75244a3b4780083462de0f9 as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/b/f9056c54b75244a3b4780083462de0f9 2024-11-23T18:49:44,847 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/b/f9056c54b75244a3b4780083462de0f9, entries=10, sequenceid=32, filesize=5.8 K 2024-11-23T18:49:44,847 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for c1c0d6f14011bf42266ffbdefd9e1ffe in 39ms, sequenceid=32, compaction requested=false; wal=null 2024-11-23T18:49:44,848 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/recovered.edits/0000000000000000032 2024-11-23T18:49:44,850 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,850 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,850 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:49:44,852 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c1c0d6f14011bf42266ffbdefd9e1ffe 2024-11-23T18:49:44,855 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/c1c0d6f14011bf42266ffbdefd9e1ffe/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-23T18:49:44,856 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c1c0d6f14011bf42266ffbdefd9e1ffe; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64669879, jitterRate=-0.036343708634376526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:49:44,857 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c1c0d6f14011bf42266ffbdefd9e1ffe: Writing region info on filesystem at 1732387784779Initializing all the Stores at 1732387784780 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387784780Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387784780Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387784780Obtaining lock to block concurrent updates at 1732387784808 (+28 ms)Preparing flush snapshotting stores in c1c0d6f14011bf42266ffbdefd9e1ffe at 1732387784809 (+1 ms)Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732387784809Flushing stores of testReplayEditsWrittenViaHRegion,,1732387784489.c1c0d6f14011bf42266ffbdefd9e1ffe. at 1732387784809Flushing c1c0d6f14011bf42266ffbdefd9e1ffe/b: creating writer at 1732387784809Flushing c1c0d6f14011bf42266ffbdefd9e1ffe/b: appending metadata at 1732387784823 (+14 ms)Flushing c1c0d6f14011bf42266ffbdefd9e1ffe/b: closing flushed file at 1732387784823Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c137001: reopening flushed file at 1732387784839 (+16 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for c1c0d6f14011bf42266ffbdefd9e1ffe in 39ms, sequenceid=32, compaction requested=false; wal=null at 1732387784847 (+8 ms)Cleaning up temporary data from old regions at 1732387784850 (+3 ms)Region opened successfully at 1732387784857 (+7 ms) 2024-11-23T18:49:44,882 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=416 (was 406) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-995492633-172.17.0.2-1732387759654:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:56518 [Waiting for operation #41] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:57606 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:38518 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:54014 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-995492633-172.17.0.2-1732387759654:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:51112 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:54792 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-995492633-172.17.0.2-1732387759654:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1118 (was 1042) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=259 (was 259), ProcessCount=11 (was 11), AvailableMemoryMB=8615 (was 8624) 2024-11-23T18:49:44,882 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1118 is superior to 1024 2024-11-23T18:49:44,897 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=416, OpenFileDescriptor=1118, MaxFileDescriptor=1048576, SystemLoadAverage=259, ProcessCount=11, AvailableMemoryMB=8614 2024-11-23T18:49:44,897 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1118 is superior to 1024 2024-11-23T18:49:44,912 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:44,914 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:44,915 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:49:44,917 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-16164838, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/hregion-16164838, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:44,929 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-16164838/hregion-16164838.1732387784917, exclude list is [], retry=0 2024-11-23T18:49:44,932 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:44,932 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:44,933 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:44,934 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-16164838/hregion-16164838.1732387784917 2024-11-23T18:49:44,935 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:39619:39619)] 2024-11-23T18:49:44,935 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2719a3ff4ddbc0c634a64b90fd98d649, NAME => 'testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43835/hbase 2024-11-23T18:49:44,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741907_1085 (size=68) 2024-11-23T18:49:44,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741907_1085 (size=68) 2024-11-23T18:49:44,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741907_1085 (size=68) 2024-11-23T18:49:44,944 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:44,945 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:44,947 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2719a3ff4ddbc0c634a64b90fd98d649 columnFamilyName a 2024-11-23T18:49:44,947 DEBUG [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:44,947 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(327): Store=2719a3ff4ddbc0c634a64b90fd98d649/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:44,948 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:44,949 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2719a3ff4ddbc0c634a64b90fd98d649 columnFamilyName b 2024-11-23T18:49:44,949 DEBUG [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:44,949 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(327): Store=2719a3ff4ddbc0c634a64b90fd98d649/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:44,949 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:44,951 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2719a3ff4ddbc0c634a64b90fd98d649 columnFamilyName c 2024-11-23T18:49:44,951 DEBUG [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:44,951 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(327): Store=2719a3ff4ddbc0c634a64b90fd98d649/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:44,951 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:44,952 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:44,952 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:44,953 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:44,953 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:44,954 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:49:44,955 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:44,957 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:49:44,957 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2719a3ff4ddbc0c634a64b90fd98d649; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61915429, jitterRate=-0.0773882120847702}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:49:44,958 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2719a3ff4ddbc0c634a64b90fd98d649: Writing region info on filesystem at 1732387784944Initializing all the Stores at 1732387784945 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387784945Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387784945Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387784945Cleaning up temporary data from old regions at 1732387784953 (+8 ms)Region opened successfully at 1732387784958 (+5 ms) 2024-11-23T18:49:44,958 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2719a3ff4ddbc0c634a64b90fd98d649, disabling compactions & flushes 2024-11-23T18:49:44,958 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649. 2024-11-23T18:49:44,958 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649. 2024-11-23T18:49:44,958 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649. after waiting 0 ms 2024-11-23T18:49:44,958 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649. 2024-11-23T18:49:44,958 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649. 2024-11-23T18:49:44,958 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2719a3ff4ddbc0c634a64b90fd98d649: Waiting for close lock at 1732387784958Disabling compacts and flushes for region at 1732387784958Disabling writes for close at 1732387784958Writing region close event to WAL at 1732387784958Closed at 1732387784958 2024-11-23T18:49:44,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741906_1084 (size=95) 2024-11-23T18:49:44,962 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-16164838/hregion-16164838.1732387784917 not finished, retry = 0 2024-11-23T18:49:44,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741906_1084 (size=95) 2024-11-23T18:49:44,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741906_1084 (size=95) 2024-11-23T18:49:45,065 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:49:45,065 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-16164838:(num 1732387784917) 2024-11-23T18:49:45,065 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:49:45,067 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:45,079 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387785067, exclude list is [], retry=0 2024-11-23T18:49:45,082 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:45,082 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:45,082 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:45,084 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387785067 2024-11-23T18:49:45,087 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:39619:39619)] 2024-11-23T18:49:45,159 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2719a3ff4ddbc0c634a64b90fd98d649, NAME => 'testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:45,161 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:45,161 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:45,162 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:45,162 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:45,164 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:45,164 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2719a3ff4ddbc0c634a64b90fd98d649 columnFamilyName a 2024-11-23T18:49:45,164 DEBUG [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:45,165 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(327): Store=2719a3ff4ddbc0c634a64b90fd98d649/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:45,165 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:45,166 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2719a3ff4ddbc0c634a64b90fd98d649 columnFamilyName b 2024-11-23T18:49:45,166 DEBUG [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:45,166 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(327): Store=2719a3ff4ddbc0c634a64b90fd98d649/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:45,167 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:45,167 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2719a3ff4ddbc0c634a64b90fd98d649 columnFamilyName c 2024-11-23T18:49:45,167 DEBUG [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:45,168 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(327): Store=2719a3ff4ddbc0c634a64b90fd98d649/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:45,168 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:45,169 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:45,170 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:45,171 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:45,171 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:45,172 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:49:45,173 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:45,174 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2719a3ff4ddbc0c634a64b90fd98d649; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74847384, jitterRate=0.11531293392181396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:49:45,174 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:45,175 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2719a3ff4ddbc0c634a64b90fd98d649: Running coprocessor pre-open hook at 1732387785162Writing region info on filesystem at 1732387785162Initializing all the Stores at 1732387785163 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387785163Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387785163Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387785163Cleaning up temporary data from old regions at 1732387785171 (+8 ms)Running coprocessor post-open hooks at 1732387785174 (+3 ms)Region opened successfully at 1732387785175 (+1 ms) 2024-11-23T18:49:45,187 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2719a3ff4ddbc0c634a64b90fd98d649 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-23T18:49:45,188 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2719a3ff4ddbc0c634a64b90fd98d649/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:45,908 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T18:49:46,189 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2719a3ff4ddbc0c634a64b90fd98d649/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:47,190 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2719a3ff4ddbc0c634a64b90fd98d649/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:48,191 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2719a3ff4ddbc0c634a64b90fd98d649/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:49,191 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2719a3ff4ddbc0c634a64b90fd98d649/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:50,192 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2719a3ff4ddbc0c634a64b90fd98d649/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:51,193 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2719a3ff4ddbc0c634a64b90fd98d649/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:52,194 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2719a3ff4ddbc0c634a64b90fd98d649/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:53,195 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2719a3ff4ddbc0c634a64b90fd98d649/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:53,424 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T18:49:54,196 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2719a3ff4ddbc0c634a64b90fd98d649/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:54,197 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2719a3ff4ddbc0c634a64b90fd98d649: 2024-11-23T18:49:54,198 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649. 2024-11-23T18:49:54,210 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2719a3ff4ddbc0c634a64b90fd98d649: 2024-11-23T18:49:54,210 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-23T18:49:54,211 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2719a3ff4ddbc0c634a64b90fd98d649, disabling compactions & flushes 2024-11-23T18:49:54,211 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649. 2024-11-23T18:49:54,211 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649. 2024-11-23T18:49:54,211 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649. after waiting 0 ms 2024-11-23T18:49:54,211 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649. 2024-11-23T18:49:54,211 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649. 2024-11-23T18:49:54,211 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649. 2024-11-23T18:49:54,211 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2719a3ff4ddbc0c634a64b90fd98d649: Waiting for close lock at 1732387794211Running coprocessor pre-close hooks at 1732387794211Disabling compacts and flushes for region at 1732387794211Disabling writes for close at 1732387794211Writing region close event to WAL at 1732387794211Running coprocessor post-close hooks at 1732387794211Closed at 1732387794211 2024-11-23T18:49:54,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741908_1086 (size=2691) 2024-11-23T18:49:54,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741908_1086 (size=2691) 2024-11-23T18:49:54,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741908_1086 (size=2691) 2024-11-23T18:49:54,236 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43835/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387785067, size=2.6 K (2691bytes) 2024-11-23T18:49:54,236 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387785067 2024-11-23T18:49:54,237 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43835/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387785067 after 1ms 2024-11-23T18:49:54,239 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387785067: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:54,239 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43835/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387785067 took 3ms 2024-11-23T18:49:54,242 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43835/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387785067 so closing down 2024-11-23T18:49:54,242 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:49:54,243 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732387785067.temp 2024-11-23T18:49:54,244 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/recovered.edits/0000000000000000004-wal.1732387785067.temp 2024-11-23T18:49:54,244 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:49:54,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741909_1087 (size=2094) 2024-11-23T18:49:54,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741909_1087 (size=2094) 2024-11-23T18:49:54,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741909_1087 (size=2094) 2024-11-23T18:49:54,255 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/recovered.edits/0000000000000000004-wal.1732387785067.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-23T18:49:54,256 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/recovered.edits/0000000000000000004-wal.1732387785067.temp to hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/recovered.edits/0000000000000000026 2024-11-23T18:49:54,256 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:43835/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387785067, size=2.6 K, length=2691, corrupted=false, cancelled=false 2024-11-23T18:49:54,256 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43835/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387785067, journal: Splitting hdfs://localhost:43835/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387785067, size=2.6 K (2691bytes) at 1732387794236Finishing writing output for hdfs://localhost:43835/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387785067 so closing down at 1732387794242 (+6 ms)Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/recovered.edits/0000000000000000004-wal.1732387785067.temp at 1732387794244 (+2 ms)3 split writer threads finished at 1732387794244Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/recovered.edits/0000000000000000004-wal.1732387785067.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732387794255 (+11 ms)Rename recovered edits hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/recovered.edits/0000000000000000004-wal.1732387785067.temp to hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/recovered.edits/0000000000000000026 at 1732387794256 (+1 ms)Processed 23 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:43835/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387785067, size=2.6 K, length=2691, corrupted=false, cancelled=false at 1732387794256 2024-11-23T18:49:54,258 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43835/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387785067 to hdfs://localhost:43835/hbase/oldWALs/wal.1732387785067 2024-11-23T18:49:54,259 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/recovered.edits/0000000000000000026 2024-11-23T18:49:54,259 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:49:54,260 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:54,274 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387794261, exclude list is [], retry=0 2024-11-23T18:49:54,277 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:54,277 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:54,278 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:54,279 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387794261 2024-11-23T18:49:54,279 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:39619:39619)] 2024-11-23T18:49:54,280 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2719a3ff4ddbc0c634a64b90fd98d649, NAME => 'testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:54,280 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:54,280 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:54,280 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:54,280 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:54,282 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:54,283 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2719a3ff4ddbc0c634a64b90fd98d649 columnFamilyName a 2024-11-23T18:49:54,283 DEBUG [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:54,284 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(327): Store=2719a3ff4ddbc0c634a64b90fd98d649/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:54,284 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:54,285 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2719a3ff4ddbc0c634a64b90fd98d649 columnFamilyName b 2024-11-23T18:49:54,285 DEBUG [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:54,286 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(327): Store=2719a3ff4ddbc0c634a64b90fd98d649/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:54,286 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:54,287 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2719a3ff4ddbc0c634a64b90fd98d649 columnFamilyName c 2024-11-23T18:49:54,287 DEBUG [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:54,287 INFO [StoreOpener-2719a3ff4ddbc0c634a64b90fd98d649-1 {}] regionserver.HStore(327): Store=2719a3ff4ddbc0c634a64b90fd98d649/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:54,288 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:54,288 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:54,290 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:54,290 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/recovered.edits/0000000000000000026 2024-11-23T18:49:54,292 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:54,294 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/recovered.edits/0000000000000000026 2024-11-23T18:49:54,294 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2719a3ff4ddbc0c634a64b90fd98d649 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-23T18:49:54,309 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/.tmp/a/d5947eb1e57d46ff9ad1f7c43a4e5f45 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732387794202/Put/seqid=0 2024-11-23T18:49:54,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741911_1089 (size=5523) 2024-11-23T18:49:54,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741911_1089 (size=5523) 2024-11-23T18:49:54,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741911_1089 (size=5523) 2024-11-23T18:49:54,315 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/.tmp/a/d5947eb1e57d46ff9ad1f7c43a4e5f45 2024-11-23T18:49:54,334 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/.tmp/b/02f7396d82944bc1a647bb448651d6d9 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732387794198/Put/seqid=0 2024-11-23T18:49:54,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741912_1090 (size=5524) 2024-11-23T18:49:54,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741912_1090 (size=5524) 2024-11-23T18:49:54,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741912_1090 (size=5524) 2024-11-23T18:49:54,344 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/.tmp/b/02f7396d82944bc1a647bb448651d6d9 2024-11-23T18:49:54,365 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/.tmp/c/cf610a906ebd4ee5af63db54d0d389cd is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732387794199/Put/seqid=0 2024-11-23T18:49:54,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741913_1091 (size=5457) 2024-11-23T18:49:54,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741913_1091 (size=5457) 2024-11-23T18:49:54,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741913_1091 (size=5457) 2024-11-23T18:49:54,372 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/.tmp/c/cf610a906ebd4ee5af63db54d0d389cd 2024-11-23T18:49:54,380 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/.tmp/a/d5947eb1e57d46ff9ad1f7c43a4e5f45 as hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/a/d5947eb1e57d46ff9ad1f7c43a4e5f45 2024-11-23T18:49:54,385 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/a/d5947eb1e57d46ff9ad1f7c43a4e5f45, entries=7, sequenceid=26, filesize=5.4 K 2024-11-23T18:49:54,387 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/.tmp/b/02f7396d82944bc1a647bb448651d6d9 as hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/b/02f7396d82944bc1a647bb448651d6d9 2024-11-23T18:49:54,393 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/b/02f7396d82944bc1a647bb448651d6d9, entries=7, sequenceid=26, filesize=5.4 K 2024-11-23T18:49:54,394 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/.tmp/c/cf610a906ebd4ee5af63db54d0d389cd as hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/c/cf610a906ebd4ee5af63db54d0d389cd 2024-11-23T18:49:54,400 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/c/cf610a906ebd4ee5af63db54d0d389cd, entries=6, sequenceid=26, filesize=5.3 K 2024-11-23T18:49:54,400 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 2719a3ff4ddbc0c634a64b90fd98d649 in 106ms, sequenceid=26, compaction requested=false; wal=null 2024-11-23T18:49:54,401 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/recovered.edits/0000000000000000026 2024-11-23T18:49:54,402 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:54,402 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:54,403 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:49:54,405 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:54,407 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testReplayEditsAfterAbortingFlush/2719a3ff4ddbc0c634a64b90fd98d649/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-23T18:49:54,408 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2719a3ff4ddbc0c634a64b90fd98d649; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75379758, jitterRate=0.12324592471122742}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:49:54,408 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2719a3ff4ddbc0c634a64b90fd98d649 2024-11-23T18:49:54,408 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2719a3ff4ddbc0c634a64b90fd98d649: Running coprocessor pre-open hook at 1732387794280Writing region info on filesystem at 1732387794280Initializing all the Stores at 1732387794281 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387794281Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387794281Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387794281Obtaining lock to block concurrent updates at 1732387794294 (+13 ms)Preparing flush snapshotting stores in 2719a3ff4ddbc0c634a64b90fd98d649 at 1732387794294Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732387794294Flushing stores of testReplayEditsAfterAbortingFlush,,1732387784913.2719a3ff4ddbc0c634a64b90fd98d649. at 1732387794294Flushing 2719a3ff4ddbc0c634a64b90fd98d649/a: creating writer at 1732387794295 (+1 ms)Flushing 2719a3ff4ddbc0c634a64b90fd98d649/a: appending metadata at 1732387794308 (+13 ms)Flushing 2719a3ff4ddbc0c634a64b90fd98d649/a: closing flushed file at 1732387794309 (+1 ms)Flushing 2719a3ff4ddbc0c634a64b90fd98d649/b: creating writer at 1732387794320 (+11 ms)Flushing 2719a3ff4ddbc0c634a64b90fd98d649/b: appending metadata at 1732387794334 (+14 ms)Flushing 2719a3ff4ddbc0c634a64b90fd98d649/b: closing flushed file at 1732387794334Flushing 2719a3ff4ddbc0c634a64b90fd98d649/c: creating writer at 1732387794350 (+16 ms)Flushing 2719a3ff4ddbc0c634a64b90fd98d649/c: appending metadata at 1732387794365 (+15 ms)Flushing 2719a3ff4ddbc0c634a64b90fd98d649/c: closing flushed file at 1732387794365Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a0d6a4c: reopening flushed file at 1732387794379 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6400722a: reopening flushed file at 1732387794386 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77e86653: reopening flushed file at 1732387794393 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 2719a3ff4ddbc0c634a64b90fd98d649 in 106ms, sequenceid=26, compaction requested=false; wal=null at 1732387794400 (+7 ms)Cleaning up temporary data from old regions at 1732387794402 (+2 ms)Running coprocessor post-open hooks at 1732387794408 (+6 ms)Region opened successfully at 1732387794408 2024-11-23T18:49:54,427 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=413 (was 416), OpenFileDescriptor=1178 (was 1118) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=235 (was 259), ProcessCount=11 (was 11), AvailableMemoryMB=8624 (was 8614) - AvailableMemoryMB LEAK? - 2024-11-23T18:49:54,427 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1178 is superior to 1024 2024-11-23T18:49:54,437 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=413, OpenFileDescriptor=1178, MaxFileDescriptor=1048576, SystemLoadAverage=235, ProcessCount=11, AvailableMemoryMB=8624 2024-11-23T18:49:54,437 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1178 is superior to 1024 2024-11-23T18:49:54,452 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:54,453 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:54,454 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:49:54,456 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-95178919, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/hregion-95178919, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:54,469 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-95178919/hregion-95178919.1732387794457, exclude list is [], retry=0 2024-11-23T18:49:54,472 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:54,472 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:54,472 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:54,474 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-95178919/hregion-95178919.1732387794457 2024-11-23T18:49:54,475 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:43381:43381)] 2024-11-23T18:49:54,475 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7e79062bbcb62bde2d1f2587df58e3aa, NAME => 'testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43835/hbase 2024-11-23T18:49:54,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741915_1093 (size=61) 2024-11-23T18:49:54,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741915_1093 (size=61) 2024-11-23T18:49:54,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741915_1093 (size=61) 2024-11-23T18:49:54,489 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:54,490 INFO [StoreOpener-7e79062bbcb62bde2d1f2587df58e3aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:54,492 INFO [StoreOpener-7e79062bbcb62bde2d1f2587df58e3aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e79062bbcb62bde2d1f2587df58e3aa columnFamilyName a 2024-11-23T18:49:54,492 DEBUG [StoreOpener-7e79062bbcb62bde2d1f2587df58e3aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:54,493 INFO [StoreOpener-7e79062bbcb62bde2d1f2587df58e3aa-1 {}] regionserver.HStore(327): Store=7e79062bbcb62bde2d1f2587df58e3aa/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:54,493 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:54,493 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:54,494 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:54,494 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:54,494 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:54,495 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:54,501 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:49:54,501 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7e79062bbcb62bde2d1f2587df58e3aa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58795960, jitterRate=-0.12387192249298096}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:49:54,502 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7e79062bbcb62bde2d1f2587df58e3aa: Writing region info on filesystem at 1732387794489Initializing all the Stores at 1732387794490 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387794490Cleaning up temporary data from old regions at 1732387794494 (+4 ms)Region opened successfully at 1732387794502 (+8 ms) 2024-11-23T18:49:54,502 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7e79062bbcb62bde2d1f2587df58e3aa, disabling compactions & flushes 2024-11-23T18:49:54,502 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa. 2024-11-23T18:49:54,502 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa. 2024-11-23T18:49:54,502 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa. after waiting 0 ms 2024-11-23T18:49:54,502 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa. 2024-11-23T18:49:54,502 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa. 2024-11-23T18:49:54,503 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7e79062bbcb62bde2d1f2587df58e3aa: Waiting for close lock at 1732387794502Disabling compacts and flushes for region at 1732387794502Disabling writes for close at 1732387794502Writing region close event to WAL at 1732387794502Closed at 1732387794502 2024-11-23T18:49:54,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741914_1092 (size=95) 2024-11-23T18:49:54,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741914_1092 (size=95) 2024-11-23T18:49:54,509 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-95178919/hregion-95178919.1732387794457 not finished, retry = 0 2024-11-23T18:49:54,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741914_1092 (size=95) 2024-11-23T18:49:54,598 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-23T18:49:54,598 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-23T18:49:54,612 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:49:54,612 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-95178919:(num 1732387794457) 2024-11-23T18:49:54,612 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:49:54,616 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:54,632 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387794617, exclude list is [], retry=0 2024-11-23T18:49:54,635 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:54,635 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:54,635 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:54,637 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387794617 2024-11-23T18:49:54,637 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:37451:37451)] 2024-11-23T18:49:54,637 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7e79062bbcb62bde2d1f2587df58e3aa, NAME => 'testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:54,637 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:54,637 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:54,637 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:54,639 INFO [StoreOpener-7e79062bbcb62bde2d1f2587df58e3aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:54,640 INFO [StoreOpener-7e79062bbcb62bde2d1f2587df58e3aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e79062bbcb62bde2d1f2587df58e3aa columnFamilyName a 2024-11-23T18:49:54,640 DEBUG [StoreOpener-7e79062bbcb62bde2d1f2587df58e3aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:54,641 INFO [StoreOpener-7e79062bbcb62bde2d1f2587df58e3aa-1 {}] regionserver.HStore(327): Store=7e79062bbcb62bde2d1f2587df58e3aa/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:54,641 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:54,642 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:54,643 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:54,643 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:54,643 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:54,645 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:54,646 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7e79062bbcb62bde2d1f2587df58e3aa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65793971, jitterRate=-0.019593432545661926}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:49:54,647 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7e79062bbcb62bde2d1f2587df58e3aa: Writing region info on filesystem at 1732387794637Initializing all the Stores at 1732387794638 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387794638Cleaning up temporary data from old regions at 1732387794643 (+5 ms)Region opened successfully at 1732387794647 (+4 ms) 2024-11-23T18:49:54,656 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7e79062bbcb62bde2d1f2587df58e3aa, disabling compactions & flushes 2024-11-23T18:49:54,657 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa. 2024-11-23T18:49:54,657 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa. 2024-11-23T18:49:54,657 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa. after waiting 0 ms 2024-11-23T18:49:54,657 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa. 2024-11-23T18:49:54,657 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa. 2024-11-23T18:49:54,657 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa. 2024-11-23T18:49:54,657 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7e79062bbcb62bde2d1f2587df58e3aa: Waiting for close lock at 1732387794656Disabling compacts and flushes for region at 1732387794656Disabling writes for close at 1732387794657 (+1 ms)Writing region close event to WAL at 1732387794657Closed at 1732387794657 2024-11-23T18:49:54,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741916_1094 (size=1050) 2024-11-23T18:49:54,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741916_1094 (size=1050) 2024-11-23T18:49:54,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741916_1094 (size=1050) 2024-11-23T18:49:54,683 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43835/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387794617, size=1.0 K (1050bytes) 2024-11-23T18:49:54,683 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387794617 2024-11-23T18:49:54,683 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43835/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387794617 after 0ms 2024-11-23T18:49:54,685 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387794617: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:54,685 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43835/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387794617 took 3ms 2024-11-23T18:49:54,688 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43835/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387794617 so closing down 2024-11-23T18:49:54,688 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:49:54,689 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732387794617.temp 2024-11-23T18:49:54,690 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/recovered.edits/0000000000000000003-wal.1732387794617.temp 2024-11-23T18:49:54,691 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:49:54,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741917_1095 (size=1050) 2024-11-23T18:49:54,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741917_1095 (size=1050) 2024-11-23T18:49:54,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741917_1095 (size=1050) 2024-11-23T18:49:54,697 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/recovered.edits/0000000000000000003-wal.1732387794617.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-23T18:49:54,698 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/recovered.edits/0000000000000000003-wal.1732387794617.temp to hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/recovered.edits/0000000000000000012 2024-11-23T18:49:54,698 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387794617, size=1.0 K, length=1050, corrupted=false, cancelled=false 2024-11-23T18:49:54,698 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43835/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387794617, journal: Splitting hdfs://localhost:43835/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387794617, size=1.0 K (1050bytes) at 1732387794683Finishing writing output for hdfs://localhost:43835/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387794617 so closing down at 1732387794688 (+5 ms)Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/recovered.edits/0000000000000000003-wal.1732387794617.temp at 1732387794690 (+2 ms)3 split writer threads finished at 1732387794691 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/recovered.edits/0000000000000000003-wal.1732387794617.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732387794697 (+6 ms)Rename recovered edits hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/recovered.edits/0000000000000000003-wal.1732387794617.temp to hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/recovered.edits/0000000000000000012 at 1732387794698 (+1 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387794617, size=1.0 K, length=1050, corrupted=false, cancelled=false at 1732387794698 2024-11-23T18:49:54,700 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43835/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387794617 to hdfs://localhost:43835/hbase/oldWALs/wal.1732387794617 2024-11-23T18:49:54,701 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/recovered.edits/0000000000000000012 2024-11-23T18:49:54,704 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:55,070 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:49:55,072 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:55,085 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387795072, exclude list is [], retry=0 2024-11-23T18:49:55,088 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:55,088 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:55,088 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:55,090 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387795072 2024-11-23T18:49:55,090 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:39619:39619)] 2024-11-23T18:49:55,090 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7e79062bbcb62bde2d1f2587df58e3aa, NAME => 'testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:55,090 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:55,090 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,090 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,093 INFO [StoreOpener-7e79062bbcb62bde2d1f2587df58e3aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,095 INFO [StoreOpener-7e79062bbcb62bde2d1f2587df58e3aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e79062bbcb62bde2d1f2587df58e3aa columnFamilyName a 2024-11-23T18:49:55,095 DEBUG [StoreOpener-7e79062bbcb62bde2d1f2587df58e3aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:55,096 INFO [StoreOpener-7e79062bbcb62bde2d1f2587df58e3aa-1 {}] regionserver.HStore(327): Store=7e79062bbcb62bde2d1f2587df58e3aa/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:55,096 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,096 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,098 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,098 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/recovered.edits/0000000000000000012 2024-11-23T18:49:55,100 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:55,101 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/recovered.edits/0000000000000000012 2024-11-23T18:49:55,101 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7e79062bbcb62bde2d1f2587df58e3aa 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-23T18:49:55,121 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/.tmp/a/75d3bf476bc3447e934f41f067ca1284 is 79, key is testDatalossWhenInputError/a:x0/1732387794647/Put/seqid=0 2024-11-23T18:49:55,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741919_1097 (size=5808) 2024-11-23T18:49:55,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741919_1097 (size=5808) 2024-11-23T18:49:55,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741919_1097 (size=5808) 2024-11-23T18:49:55,128 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/.tmp/a/75d3bf476bc3447e934f41f067ca1284 2024-11-23T18:49:55,136 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/.tmp/a/75d3bf476bc3447e934f41f067ca1284 as hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/a/75d3bf476bc3447e934f41f067ca1284 2024-11-23T18:49:55,144 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/a/75d3bf476bc3447e934f41f067ca1284, entries=10, sequenceid=12, filesize=5.7 K 2024-11-23T18:49:55,144 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 7e79062bbcb62bde2d1f2587df58e3aa in 43ms, sequenceid=12, compaction requested=false; wal=null 2024-11-23T18:49:55,145 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/recovered.edits/0000000000000000012 2024-11-23T18:49:55,146 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,146 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,149 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,151 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-23T18:49:55,152 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7e79062bbcb62bde2d1f2587df58e3aa; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63445413, jitterRate=-0.05458967387676239}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:49:55,153 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7e79062bbcb62bde2d1f2587df58e3aa: Writing region info on filesystem at 1732387795091Initializing all the Stores at 1732387795093 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387795093Obtaining lock to block concurrent updates at 1732387795101 (+8 ms)Preparing flush snapshotting stores in 7e79062bbcb62bde2d1f2587df58e3aa at 1732387795101Finished memstore snapshotting testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732387795101Flushing stores of testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa. at 1732387795101Flushing 7e79062bbcb62bde2d1f2587df58e3aa/a: creating writer at 1732387795101Flushing 7e79062bbcb62bde2d1f2587df58e3aa/a: appending metadata at 1732387795120 (+19 ms)Flushing 7e79062bbcb62bde2d1f2587df58e3aa/a: closing flushed file at 1732387795120Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bcd6c: reopening flushed file at 1732387795135 (+15 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 7e79062bbcb62bde2d1f2587df58e3aa in 43ms, sequenceid=12, compaction requested=false; wal=null at 1732387795144 (+9 ms)Cleaning up temporary data from old regions at 1732387795146 (+2 ms)Region opened successfully at 1732387795153 (+7 ms) 2024-11-23T18:49:55,157 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7e79062bbcb62bde2d1f2587df58e3aa, NAME => 'testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:55,157 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732387794452.7e79062bbcb62bde2d1f2587df58e3aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:55,157 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,157 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,158 INFO [StoreOpener-7e79062bbcb62bde2d1f2587df58e3aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,159 INFO [StoreOpener-7e79062bbcb62bde2d1f2587df58e3aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e79062bbcb62bde2d1f2587df58e3aa columnFamilyName a 2024-11-23T18:49:55,159 DEBUG [StoreOpener-7e79062bbcb62bde2d1f2587df58e3aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:55,164 DEBUG [StoreOpener-7e79062bbcb62bde2d1f2587df58e3aa-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/a/75d3bf476bc3447e934f41f067ca1284 2024-11-23T18:49:55,164 INFO [StoreOpener-7e79062bbcb62bde2d1f2587df58e3aa-1 {}] regionserver.HStore(327): Store=7e79062bbcb62bde2d1f2587df58e3aa/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:55,164 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,165 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,166 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,167 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,167 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,168 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7e79062bbcb62bde2d1f2587df58e3aa 2024-11-23T18:49:55,170 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testDatalossWhenInputError/7e79062bbcb62bde2d1f2587df58e3aa/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-23T18:49:55,171 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7e79062bbcb62bde2d1f2587df58e3aa; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63923915, jitterRate=-0.0474594384431839}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:49:55,171 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7e79062bbcb62bde2d1f2587df58e3aa: Writing region info on filesystem at 1732387795157Initializing all the Stores at 1732387795158 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387795158Cleaning up temporary data from old regions at 1732387795167 (+9 ms)Region opened successfully at 1732387795171 (+4 ms) 2024-11-23T18:49:55,187 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=423 (was 413) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-995492633-172.17.0.2-1732387759654:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:52762 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-995492633-172.17.0.2-1732387759654:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:52810 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:48334 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:48244 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:46270 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-995492633-172.17.0.2-1732387759654:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:46344 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1260 (was 1178) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=235 (was 235), ProcessCount=11 (was 11), AvailableMemoryMB=8615 (was 8624) 2024-11-23T18:49:55,187 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1260 is superior to 1024 2024-11-23T18:49:55,201 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=423, OpenFileDescriptor=1260, MaxFileDescriptor=1048576, SystemLoadAverage=235, ProcessCount=11, AvailableMemoryMB=8614 2024-11-23T18:49:55,201 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1260 is superior to 1024 2024-11-23T18:49:55,218 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:55,219 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:49:55,220 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:49:55,222 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-67538280, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/hregion-67538280, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:55,234 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-67538280/hregion-67538280.1732387795223, exclude list is [], retry=0 2024-11-23T18:49:55,237 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:55,237 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:55,237 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:55,239 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-67538280/hregion-67538280.1732387795223 2024-11-23T18:49:55,240 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:43381:43381)] 2024-11-23T18:49:55,240 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f0ad5be90fc45715cd9d587a9881295b, NAME => 'testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43835/hbase 2024-11-23T18:49:55,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741921_1099 (size=63) 2024-11-23T18:49:55,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741921_1099 (size=63) 2024-11-23T18:49:55,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741921_1099 (size=63) 2024-11-23T18:49:55,650 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:55,652 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,655 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0ad5be90fc45715cd9d587a9881295b columnFamilyName a 2024-11-23T18:49:55,655 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:55,656 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(327): Store=f0ad5be90fc45715cd9d587a9881295b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:55,656 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,658 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0ad5be90fc45715cd9d587a9881295b columnFamilyName b 2024-11-23T18:49:55,658 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:55,659 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(327): Store=f0ad5be90fc45715cd9d587a9881295b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:55,659 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,661 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0ad5be90fc45715cd9d587a9881295b columnFamilyName c 2024-11-23T18:49:55,661 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:55,661 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(327): Store=f0ad5be90fc45715cd9d587a9881295b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:55,662 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,663 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,663 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,665 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,665 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,666 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:49:55,668 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,670 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:49:55,671 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f0ad5be90fc45715cd9d587a9881295b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72807628, jitterRate=0.08491820096969604}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:49:55,671 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f0ad5be90fc45715cd9d587a9881295b: Writing region info on filesystem at 1732387795650Initializing all the Stores at 1732387795652 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387795652Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387795652Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387795652Cleaning up temporary data from old regions at 1732387795665 (+13 ms)Region opened successfully at 1732387795671 (+6 ms) 2024-11-23T18:49:55,672 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f0ad5be90fc45715cd9d587a9881295b, disabling compactions & flushes 2024-11-23T18:49:55,672 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b. 2024-11-23T18:49:55,672 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b. 2024-11-23T18:49:55,672 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b. after waiting 0 ms 2024-11-23T18:49:55,672 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b. 2024-11-23T18:49:55,672 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b. 2024-11-23T18:49:55,672 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f0ad5be90fc45715cd9d587a9881295b: Waiting for close lock at 1732387795672Disabling compacts and flushes for region at 1732387795672Disabling writes for close at 1732387795672Writing region close event to WAL at 1732387795672Closed at 1732387795672 2024-11-23T18:49:55,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741920_1098 (size=95) 2024-11-23T18:49:55,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741920_1098 (size=95) 2024-11-23T18:49:55,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741920_1098 (size=95) 2024-11-23T18:49:55,677 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:49:55,677 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-67538280:(num 1732387795223) 2024-11-23T18:49:55,677 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:49:55,679 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:55,692 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679, exclude list is [], retry=0 2024-11-23T18:49:55,694 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:55,695 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:55,695 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:55,696 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679 2024-11-23T18:49:55,697 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:43381:43381)] 2024-11-23T18:49:55,697 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f0ad5be90fc45715cd9d587a9881295b, NAME => 'testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:55,697 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:55,697 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,697 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,698 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,699 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0ad5be90fc45715cd9d587a9881295b columnFamilyName a 2024-11-23T18:49:55,699 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:55,700 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(327): Store=f0ad5be90fc45715cd9d587a9881295b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:55,700 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,700 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0ad5be90fc45715cd9d587a9881295b columnFamilyName b 2024-11-23T18:49:55,700 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:55,701 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(327): Store=f0ad5be90fc45715cd9d587a9881295b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:55,701 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,701 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0ad5be90fc45715cd9d587a9881295b columnFamilyName c 2024-11-23T18:49:55,702 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:55,702 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(327): Store=f0ad5be90fc45715cd9d587a9881295b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:55,702 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,703 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,704 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,705 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,705 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,705 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:49:55,706 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:55,707 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f0ad5be90fc45715cd9d587a9881295b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70727788, jitterRate=0.053926169872283936}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:49:55,708 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f0ad5be90fc45715cd9d587a9881295b: Writing region info on filesystem at 1732387795697Initializing all the Stores at 1732387795698 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387795698Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387795698Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387795698Cleaning up temporary data from old regions at 1732387795705 (+7 ms)Region opened successfully at 1732387795708 (+3 ms) 2024-11-23T18:49:55,712 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732387795712/Put/seqid=0 2024-11-23T18:49:55,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741923_1101 (size=4875) 2024-11-23T18:49:55,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741923_1101 (size=4875) 2024-11-23T18:49:55,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741923_1101 (size=4875) 2024-11-23T18:49:55,721 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732387795721/Put/seqid=0 2024-11-23T18:49:55,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741924_1102 (size=4875) 2024-11-23T18:49:55,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741924_1102 (size=4875) 2024-11-23T18:49:55,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741924_1102 (size=4875) 2024-11-23T18:49:55,732 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732387795732/Put/seqid=0 2024-11-23T18:49:55,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741925_1103 (size=4875) 2024-11-23T18:49:55,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741925_1103 (size=4875) 2024-11-23T18:49:55,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741925_1103 (size=4875) 2024-11-23T18:49:55,740 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43835/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in f0ad5be90fc45715cd9d587a9881295b/a 2024-11-23T18:49:55,743 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-23T18:49:55,743 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-23T18:49:55,743 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43835/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in f0ad5be90fc45715cd9d587a9881295b/a 2024-11-23T18:49:55,746 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-23T18:49:55,746 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-23T18:49:55,747 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43835/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in f0ad5be90fc45715cd9d587a9881295b/a 2024-11-23T18:49:55,750 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-23T18:49:55,750 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-23T18:49:55,750 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f0ad5be90fc45715cd9d587a9881295b 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-23T18:49:55,765 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/.tmp/a/7ce0fe0adf92490d882fad1fca993c2b is 55, key is testCompactedBulkLoadedFiles/a:a/1732387795708/Put/seqid=0 2024-11-23T18:49:55,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741926_1104 (size=5107) 2024-11-23T18:49:55,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741926_1104 (size=5107) 2024-11-23T18:49:55,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741926_1104 (size=5107) 2024-11-23T18:49:55,771 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/.tmp/a/7ce0fe0adf92490d882fad1fca993c2b 2024-11-23T18:49:55,777 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/.tmp/a/7ce0fe0adf92490d882fad1fca993c2b as hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/7ce0fe0adf92490d882fad1fca993c2b 2024-11-23T18:49:55,782 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/7ce0fe0adf92490d882fad1fca993c2b, entries=1, sequenceid=4, filesize=5.0 K 2024-11-23T18:49:55,783 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for f0ad5be90fc45715cd9d587a9881295b in 33ms, sequenceid=4, compaction requested=false 2024-11-23T18:49:55,783 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f0ad5be90fc45715cd9d587a9881295b: 2024-11-23T18:49:55,784 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/356b859943cc43e1822fa323f484ca4b_SeqId_4_ 2024-11-23T18:49:55,785 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/608c120d0b6649e295058c2a31867c87_SeqId_4_ 2024-11-23T18:49:55,786 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/eb56c2e402d04641a89e32641ead723d_SeqId_4_ 2024-11-23T18:49:55,787 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43835/hbase/testCompactedBulkLoadedFiles/hfile0 into f0ad5be90fc45715cd9d587a9881295b/a as hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/356b859943cc43e1822fa323f484ca4b_SeqId_4_ - updating store file list. 2024-11-23T18:49:55,791 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 356b859943cc43e1822fa323f484ca4b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T18:49:55,791 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/356b859943cc43e1822fa323f484ca4b_SeqId_4_ into f0ad5be90fc45715cd9d587a9881295b/a 2024-11-23T18:49:55,791 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43835/hbase/testCompactedBulkLoadedFiles/hfile0 into f0ad5be90fc45715cd9d587a9881295b/a (new location: hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/356b859943cc43e1822fa323f484ca4b_SeqId_4_) 2024-11-23T18:49:55,792 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43835/hbase/testCompactedBulkLoadedFiles/hfile1 into f0ad5be90fc45715cd9d587a9881295b/a as hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/608c120d0b6649e295058c2a31867c87_SeqId_4_ - updating store file list. 2024-11-23T18:49:55,796 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 608c120d0b6649e295058c2a31867c87_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T18:49:55,796 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/608c120d0b6649e295058c2a31867c87_SeqId_4_ into f0ad5be90fc45715cd9d587a9881295b/a 2024-11-23T18:49:55,796 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43835/hbase/testCompactedBulkLoadedFiles/hfile1 into f0ad5be90fc45715cd9d587a9881295b/a (new location: hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/608c120d0b6649e295058c2a31867c87_SeqId_4_) 2024-11-23T18:49:55,797 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43835/hbase/testCompactedBulkLoadedFiles/hfile2 into f0ad5be90fc45715cd9d587a9881295b/a as hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/eb56c2e402d04641a89e32641ead723d_SeqId_4_ - updating store file list. 2024-11-23T18:49:55,801 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for eb56c2e402d04641a89e32641ead723d_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T18:49:55,801 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/eb56c2e402d04641a89e32641ead723d_SeqId_4_ into f0ad5be90fc45715cd9d587a9881295b/a 2024-11-23T18:49:55,801 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43835/hbase/testCompactedBulkLoadedFiles/hfile2 into f0ad5be90fc45715cd9d587a9881295b/a (new location: hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/eb56c2e402d04641a89e32641ead723d_SeqId_4_) 2024-11-23T18:49:55,808 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T18:49:55,808 DEBUG [Time-limited test {}] regionserver.HStore(1541): f0ad5be90fc45715cd9d587a9881295b/a is initiating major compaction (all files) 2024-11-23T18:49:55,808 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of f0ad5be90fc45715cd9d587a9881295b/a in testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b. 2024-11-23T18:49:55,808 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/7ce0fe0adf92490d882fad1fca993c2b, hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/356b859943cc43e1822fa323f484ca4b_SeqId_4_, hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/608c120d0b6649e295058c2a31867c87_SeqId_4_, hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/eb56c2e402d04641a89e32641ead723d_SeqId_4_] into tmpdir=hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/.tmp, totalSize=19.3 K 2024-11-23T18:49:55,809 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7ce0fe0adf92490d882fad1fca993c2b, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732387795708 2024-11-23T18:49:55,809 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 356b859943cc43e1822fa323f484ca4b_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-23T18:49:55,809 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 608c120d0b6649e295058c2a31867c87_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-23T18:49:55,810 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting eb56c2e402d04641a89e32641ead723d_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-23T18:49:55,822 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/.tmp/a/f5097ccb96614b6ca31fbba32e76c32e is 55, key is testCompactedBulkLoadedFiles/a:a/1732387795708/Put/seqid=0 2024-11-23T18:49:55,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741927_1105 (size=6154) 2024-11-23T18:49:55,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741927_1105 (size=6154) 2024-11-23T18:49:55,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741927_1105 (size=6154) 2024-11-23T18:49:55,833 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/.tmp/a/f5097ccb96614b6ca31fbba32e76c32e as hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/f5097ccb96614b6ca31fbba32e76c32e 2024-11-23T18:49:55,840 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in f0ad5be90fc45715cd9d587a9881295b/a of f0ad5be90fc45715cd9d587a9881295b into f5097ccb96614b6ca31fbba32e76c32e(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:49:55,840 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for f0ad5be90fc45715cd9d587a9881295b: 2024-11-23T18:49:55,840 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-23T18:49:55,840 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-23T18:49:55,870 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43835/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679, size=0 (0bytes) 2024-11-23T18:49:55,870 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43835/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679 might be still open, length is 0 2024-11-23T18:49:55,870 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679 2024-11-23T18:49:55,871 WARN [IPC Server handler 4 on default port 43835 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-23T18:49:55,871 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43835/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679 after 1ms 2024-11-23T18:49:56,322 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:46366 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:40877:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46366 dst: /127.0.0.1:40877 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40877 remote=/127.0.0.1:46366]. Total timeout mills is 60000, 59518 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:56,322 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:52848 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:43523:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52848 dst: /127.0.0.1:43523 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:56,322 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:48380 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:37657:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48380 dst: /127.0.0.1:37657 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:49:56,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741922_1106 (size=1178) 2024-11-23T18:49:56,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741922_1106 (size=1178) 2024-11-23T18:49:56,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741922_1106 (size=1178) 2024-11-23T18:49:59,744 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T18:49:59,872 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43835/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679 after 4002ms 2024-11-23T18:49:59,877 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:59,878 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43835/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679 took 4008ms 2024-11-23T18:49:59,881 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43835/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679; continuing. 2024-11-23T18:49:59,881 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43835/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679 so closing down 2024-11-23T18:49:59,881 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:49:59,884 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732387795679.temp 2024-11-23T18:49:59,885 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/recovered.edits/0000000000000000003-wal.1732387795679.temp 2024-11-23T18:49:59,885 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:49:59,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741928_1107 (size=552) 2024-11-23T18:49:59,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741928_1107 (size=552) 2024-11-23T18:49:59,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741928_1107 (size=552) 2024-11-23T18:49:59,892 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/recovered.edits/0000000000000000003-wal.1732387795679.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-23T18:49:59,894 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/recovered.edits/0000000000000000003-wal.1732387795679.temp to hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/recovered.edits/0000000000000000008 2024-11-23T18:49:59,894 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:43835/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679, size=0, length=0, corrupted=false, cancelled=false 2024-11-23T18:49:59,894 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43835/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679, journal: Splitting hdfs://localhost:43835/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679, size=0 (0bytes) at 1732387795870Finishing writing output for hdfs://localhost:43835/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679 so closing down at 1732387799881 (+4011 ms)Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/recovered.edits/0000000000000000003-wal.1732387795679.temp at 1732387799885 (+4 ms)3 split writer threads finished at 1732387799885Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/recovered.edits/0000000000000000003-wal.1732387795679.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732387799892 (+7 ms)Rename recovered edits hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/recovered.edits/0000000000000000003-wal.1732387795679.temp to hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/recovered.edits/0000000000000000008 at 1732387799894 (+2 ms)Processed 5 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:43835/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679, size=0, length=0, corrupted=false, cancelled=false at 1732387799894 2024-11-23T18:49:59,895 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43835/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679 to hdfs://localhost:43835/hbase/oldWALs/wal.1732387795679 2024-11-23T18:49:59,896 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/recovered.edits/0000000000000000008 2024-11-23T18:49:59,896 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:49:59,898 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:49:59,910 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387799898, exclude list is [], retry=0 2024-11-23T18:49:59,912 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:49:59,913 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:49:59,913 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:49:59,914 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387799898 2024-11-23T18:49:59,915 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:39619:39619)] 2024-11-23T18:49:59,915 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f0ad5be90fc45715cd9d587a9881295b, NAME => 'testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:49:59,915 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:49:59,915 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:59,915 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:59,916 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:59,917 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0ad5be90fc45715cd9d587a9881295b columnFamilyName a 2024-11-23T18:49:59,917 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:59,923 DEBUG [StoreFileOpener-f0ad5be90fc45715cd9d587a9881295b-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 356b859943cc43e1822fa323f484ca4b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T18:49:59,923 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/356b859943cc43e1822fa323f484ca4b_SeqId_4_ 2024-11-23T18:49:59,927 DEBUG [StoreFileOpener-f0ad5be90fc45715cd9d587a9881295b-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 608c120d0b6649e295058c2a31867c87_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T18:49:59,928 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/608c120d0b6649e295058c2a31867c87_SeqId_4_ 2024-11-23T18:49:59,932 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/7ce0fe0adf92490d882fad1fca993c2b 2024-11-23T18:49:59,936 DEBUG [StoreFileOpener-f0ad5be90fc45715cd9d587a9881295b-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for eb56c2e402d04641a89e32641ead723d_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T18:49:59,936 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/eb56c2e402d04641a89e32641ead723d_SeqId_4_ 2024-11-23T18:49:59,939 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/f5097ccb96614b6ca31fbba32e76c32e 2024-11-23T18:49:59,940 WARN [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/356b859943cc43e1822fa323f484ca4b_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@11136be4 2024-11-23T18:49:59,940 WARN [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/608c120d0b6649e295058c2a31867c87_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@11136be4 2024-11-23T18:49:59,940 WARN [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/7ce0fe0adf92490d882fad1fca993c2b from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@11136be4 2024-11-23T18:49:59,940 WARN [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/eb56c2e402d04641a89e32641ead723d_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@11136be4 2024-11-23T18:49:59,940 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/356b859943cc43e1822fa323f484ca4b_SeqId_4_, hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/608c120d0b6649e295058c2a31867c87_SeqId_4_, hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/7ce0fe0adf92490d882fad1fca993c2b, hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/eb56c2e402d04641a89e32641ead723d_SeqId_4_] to archive 2024-11-23T18:49:59,941 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T18:49:59,942 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/356b859943cc43e1822fa323f484ca4b_SeqId_4_ to hdfs://localhost:43835/hbase/archive/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/356b859943cc43e1822fa323f484ca4b_SeqId_4_ 2024-11-23T18:49:59,943 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/608c120d0b6649e295058c2a31867c87_SeqId_4_ to hdfs://localhost:43835/hbase/archive/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/608c120d0b6649e295058c2a31867c87_SeqId_4_ 2024-11-23T18:49:59,944 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/7ce0fe0adf92490d882fad1fca993c2b to hdfs://localhost:43835/hbase/archive/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/7ce0fe0adf92490d882fad1fca993c2b 2024-11-23T18:49:59,945 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/eb56c2e402d04641a89e32641ead723d_SeqId_4_ to hdfs://localhost:43835/hbase/archive/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/a/eb56c2e402d04641a89e32641ead723d_SeqId_4_ 2024-11-23T18:49:59,945 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(327): Store=f0ad5be90fc45715cd9d587a9881295b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:59,946 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:59,946 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0ad5be90fc45715cd9d587a9881295b columnFamilyName b 2024-11-23T18:49:59,946 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:59,947 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(327): Store=f0ad5be90fc45715cd9d587a9881295b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:59,947 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:59,947 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0ad5be90fc45715cd9d587a9881295b columnFamilyName c 2024-11-23T18:49:59,947 DEBUG [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:49:59,948 INFO [StoreOpener-f0ad5be90fc45715cd9d587a9881295b-1 {}] regionserver.HStore(327): Store=f0ad5be90fc45715cd9d587a9881295b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:49:59,948 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:59,949 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:59,950 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:59,950 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/recovered.edits/0000000000000000008 2024-11-23T18:49:59,952 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:49:59,953 DEBUG [Time-limited test {}] regionserver.HRegion(5836): f0ad5be90fc45715cd9d587a9881295b : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "f0ad5be90fc45715cd9d587a9881295b" family_name: "a" compaction_input: "7ce0fe0adf92490d882fad1fca993c2b" compaction_input: "356b859943cc43e1822fa323f484ca4b_SeqId_4_" compaction_input: "608c120d0b6649e295058c2a31867c87_SeqId_4_" compaction_input: "eb56c2e402d04641a89e32641ead723d_SeqId_4_" compaction_output: "f5097ccb96614b6ca31fbba32e76c32e" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-23T18:49:59,954 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-23T18:49:59,954 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/recovered.edits/0000000000000000008 2024-11-23T18:49:59,954 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/recovered.edits/0000000000000000008 2024-11-23T18:49:59,955 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:59,955 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:59,956 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:49:59,957 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f0ad5be90fc45715cd9d587a9881295b 2024-11-23T18:49:59,959 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testCompactedBulkLoadedFiles/f0ad5be90fc45715cd9d587a9881295b/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-23T18:49:59,960 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f0ad5be90fc45715cd9d587a9881295b; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70880695, jitterRate=0.0562046617269516}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:49:59,960 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f0ad5be90fc45715cd9d587a9881295b: Writing region info on filesystem at 1732387799915Initializing all the Stores at 1732387799916 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387799916Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387799916Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387799916Cleaning up temporary data from old regions at 1732387799955 (+39 ms)Region opened successfully at 1732387799960 (+5 ms) 2024-11-23T18:49:59,962 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f0ad5be90fc45715cd9d587a9881295b, disabling compactions & flushes 2024-11-23T18:49:59,962 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b. 2024-11-23T18:49:59,962 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b. 2024-11-23T18:49:59,962 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b. after waiting 0 ms 2024-11-23T18:49:59,962 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b. 2024-11-23T18:49:59,963 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732387795218.f0ad5be90fc45715cd9d587a9881295b. 2024-11-23T18:49:59,963 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f0ad5be90fc45715cd9d587a9881295b: Waiting for close lock at 1732387799962Disabling compacts and flushes for region at 1732387799962Disabling writes for close at 1732387799962Writing region close event to WAL at 1732387799963 (+1 ms)Closed at 1732387799963 2024-11-23T18:49:59,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741929_1108 (size=95) 2024-11-23T18:49:59,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741929_1108 (size=95) 2024-11-23T18:49:59,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741929_1108 (size=95) 2024-11-23T18:49:59,968 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:49:59,968 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732387799898) 2024-11-23T18:49:59,982 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=437 (was 423) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2081373197_22 at /127.0.0.1:48448 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2081373197_22 at /127.0.0.1:52910 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2081373197_22 at /127.0.0.1:46428 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:43835 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:43835 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1342 (was 1260) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=232 (was 235), ProcessCount=11 (was 11), AvailableMemoryMB=8601 (was 8614) 2024-11-23T18:49:59,982 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1342 is superior to 1024 2024-11-23T18:49:59,994 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=437, OpenFileDescriptor=1342, MaxFileDescriptor=1048576, SystemLoadAverage=232, ProcessCount=11, AvailableMemoryMB=8600 2024-11-23T18:49:59,994 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1342 is superior to 1024 2024-11-23T18:50:00,008 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:50:00,010 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:50:00,011 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:50:00,013 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-22646303, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/hregion-22646303, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:50:00,026 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-22646303/hregion-22646303.1732387800014, exclude list is [], retry=0 2024-11-23T18:50:00,028 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:50:00,028 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:50:00,028 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:50:00,030 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-22646303/hregion-22646303.1732387800014 2024-11-23T18:50:00,030 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:43381:43381)] 2024-11-23T18:50:00,030 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 603528e8a0ef8db307680413e0287af5, NAME => 'testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43835/hbase 2024-11-23T18:50:00,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741931_1110 (size=67) 2024-11-23T18:50:00,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741931_1110 (size=67) 2024-11-23T18:50:00,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741931_1110 (size=67) 2024-11-23T18:50:00,039 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:50:00,040 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,042 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 603528e8a0ef8db307680413e0287af5 columnFamilyName a 2024-11-23T18:50:00,042 DEBUG [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:00,042 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(327): Store=603528e8a0ef8db307680413e0287af5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:50:00,042 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,043 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 603528e8a0ef8db307680413e0287af5 columnFamilyName b 2024-11-23T18:50:00,043 DEBUG [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:00,044 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(327): Store=603528e8a0ef8db307680413e0287af5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:50:00,044 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,045 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 603528e8a0ef8db307680413e0287af5 columnFamilyName c 2024-11-23T18:50:00,045 DEBUG [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:00,046 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(327): Store=603528e8a0ef8db307680413e0287af5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:50:00,046 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,046 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,047 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,048 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,048 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,048 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:50:00,049 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,052 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:50:00,052 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 603528e8a0ef8db307680413e0287af5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68005622, jitterRate=0.013362735509872437}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:50:00,052 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 603528e8a0ef8db307680413e0287af5: Writing region info on filesystem at 1732387800039Initializing all the Stores at 1732387800040 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387800040Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387800040Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387800040Cleaning up temporary data from old regions at 1732387800048 (+8 ms)Region opened successfully at 1732387800052 (+4 ms) 2024-11-23T18:50:00,052 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 603528e8a0ef8db307680413e0287af5, disabling compactions & flushes 2024-11-23T18:50:00,052 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. 2024-11-23T18:50:00,053 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. 2024-11-23T18:50:00,053 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. after waiting 0 ms 2024-11-23T18:50:00,053 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. 2024-11-23T18:50:00,053 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. 2024-11-23T18:50:00,053 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 603528e8a0ef8db307680413e0287af5: Waiting for close lock at 1732387800052Disabling compacts and flushes for region at 1732387800052Disabling writes for close at 1732387800053 (+1 ms)Writing region close event to WAL at 1732387800053Closed at 1732387800053 2024-11-23T18:50:00,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741930_1109 (size=95) 2024-11-23T18:50:00,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741930_1109 (size=95) 2024-11-23T18:50:00,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741930_1109 (size=95) 2024-11-23T18:50:00,057 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:50:00,058 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-22646303:(num 1732387800014) 2024-11-23T18:50:00,058 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:50:00,059 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:50:00,073 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800060, exclude list is [], retry=0 2024-11-23T18:50:00,076 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:50:00,076 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:50:00,076 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:50:00,078 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800060 2024-11-23T18:50:00,078 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:37451:37451)] 2024-11-23T18:50:00,078 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 603528e8a0ef8db307680413e0287af5, NAME => 'testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:50:00,078 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:50:00,078 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,079 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,080 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,081 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 603528e8a0ef8db307680413e0287af5 columnFamilyName a 2024-11-23T18:50:00,081 DEBUG [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:00,081 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(327): Store=603528e8a0ef8db307680413e0287af5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:50:00,082 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,082 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 603528e8a0ef8db307680413e0287af5 columnFamilyName b 2024-11-23T18:50:00,082 DEBUG [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:00,083 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(327): Store=603528e8a0ef8db307680413e0287af5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:50:00,083 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,084 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 603528e8a0ef8db307680413e0287af5 columnFamilyName c 2024-11-23T18:50:00,084 DEBUG [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:00,085 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(327): Store=603528e8a0ef8db307680413e0287af5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:50:00,085 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,086 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,087 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,087 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,087 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,088 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:50:00,089 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,090 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 603528e8a0ef8db307680413e0287af5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64047330, jitterRate=-0.04562041163444519}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:50:00,090 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 603528e8a0ef8db307680413e0287af5: Writing region info on filesystem at 1732387800079Initializing all the Stores at 1732387800080 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387800080Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387800080Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387800080Cleaning up temporary data from old regions at 1732387800087 (+7 ms)Region opened successfully at 1732387800090 (+3 ms) 2024-11-23T18:50:00,097 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 603528e8a0ef8db307680413e0287af5 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-23T18:50:00,112 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/a/8660d36b7ca04dbda4addb6b66aac6fb is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732387800090/Put/seqid=0 2024-11-23T18:50:00,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741933_1112 (size=5958) 2024-11-23T18:50:00,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741933_1112 (size=5958) 2024-11-23T18:50:00,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741933_1112 (size=5958) 2024-11-23T18:50:00,120 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/a/8660d36b7ca04dbda4addb6b66aac6fb 2024-11-23T18:50:00,126 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/a/8660d36b7ca04dbda4addb6b66aac6fb as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/a/8660d36b7ca04dbda4addb6b66aac6fb 2024-11-23T18:50:00,131 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/a/8660d36b7ca04dbda4addb6b66aac6fb, entries=10, sequenceid=13, filesize=5.8 K 2024-11-23T18:50:00,133 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 603528e8a0ef8db307680413e0287af5 in 35ms, sequenceid=13, compaction requested=false 2024-11-23T18:50:00,133 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 603528e8a0ef8db307680413e0287af5: 2024-11-23T18:50:00,148 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 603528e8a0ef8db307680413e0287af5, disabling compactions & flushes 2024-11-23T18:50:00,148 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. 2024-11-23T18:50:00,148 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. 2024-11-23T18:50:00,148 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. after waiting 0 ms 2024-11-23T18:50:00,148 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. 2024-11-23T18:50:00,149 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. 2024-11-23T18:50:00,149 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. 2024-11-23T18:50:00,149 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 603528e8a0ef8db307680413e0287af5: Waiting for close lock at 1732387800148Disabling compacts and flushes for region at 1732387800148Disabling writes for close at 1732387800148Writing region close event to WAL at 1732387800149 (+1 ms)Closed at 1732387800149 2024-11-23T18:50:00,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741932_1111 (size=3342) 2024-11-23T18:50:00,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741932_1111 (size=3342) 2024-11-23T18:50:00,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741932_1111 (size=3342) 2024-11-23T18:50:00,167 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800060, size=3.3 K (3342bytes) 2024-11-23T18:50:00,167 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800060 2024-11-23T18:50:00,167 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800060 after 0ms 2024-11-23T18:50:00,170 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800060: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:50:00,170 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800060 took 3ms 2024-11-23T18:50:00,172 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800060 so closing down 2024-11-23T18:50:00,172 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:50:00,173 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732387800060.temp 2024-11-23T18:50:00,174 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000003-wal.1732387800060.temp 2024-11-23T18:50:00,175 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:50:00,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741934_1113 (size=2944) 2024-11-23T18:50:00,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741934_1113 (size=2944) 2024-11-23T18:50:00,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741934_1113 (size=2944) 2024-11-23T18:50:00,181 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000003-wal.1732387800060.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-23T18:50:00,182 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000003-wal.1732387800060.temp to hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000035 2024-11-23T18:50:00,182 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800060, size=3.3 K, length=3342, corrupted=false, cancelled=false 2024-11-23T18:50:00,182 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800060, journal: Splitting hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800060, size=3.3 K (3342bytes) at 1732387800167Finishing writing output for hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800060 so closing down at 1732387800172 (+5 ms)Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000003-wal.1732387800060.temp at 1732387800174 (+2 ms)3 split writer threads finished at 1732387800175 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000003-wal.1732387800060.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732387800181 (+6 ms)Rename recovered edits hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000003-wal.1732387800060.temp to hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000035 at 1732387800182 (+1 ms)Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800060, size=3.3 K, length=3342, corrupted=false, cancelled=false at 1732387800182 2024-11-23T18:50:00,184 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800060 to hdfs://localhost:43835/hbase/oldWALs/wal.1732387800060 2024-11-23T18:50:00,185 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000035 2024-11-23T18:50:00,185 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:50:00,187 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:50:00,200 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187, exclude list is [], retry=0 2024-11-23T18:50:00,202 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:50:00,202 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:50:00,202 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:50:00,204 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187 2024-11-23T18:50:00,204 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37451:37451),(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:43381:43381)] 2024-11-23T18:50:00,204 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 603528e8a0ef8db307680413e0287af5, NAME => 'testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:50:00,204 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:50:00,204 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,204 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,206 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,206 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 603528e8a0ef8db307680413e0287af5 columnFamilyName a 2024-11-23T18:50:00,206 DEBUG [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:00,233 DEBUG [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/a/8660d36b7ca04dbda4addb6b66aac6fb 2024-11-23T18:50:00,233 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(327): Store=603528e8a0ef8db307680413e0287af5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:50:00,233 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,234 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 603528e8a0ef8db307680413e0287af5 columnFamilyName b 2024-11-23T18:50:00,234 DEBUG [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:00,235 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(327): Store=603528e8a0ef8db307680413e0287af5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:50:00,235 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,236 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 603528e8a0ef8db307680413e0287af5 columnFamilyName c 2024-11-23T18:50:00,236 DEBUG [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:00,236 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(327): Store=603528e8a0ef8db307680413e0287af5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:50:00,236 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,237 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,238 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,239 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000035 2024-11-23T18:50:00,241 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:50:00,242 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000035 2024-11-23T18:50:00,242 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 603528e8a0ef8db307680413e0287af5 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-23T18:50:00,257 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/b/876e96b9fc864a9c95c1ebdf3243ebea is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732387800133/Put/seqid=0 2024-11-23T18:50:00,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741936_1115 (size=5958) 2024-11-23T18:50:00,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741936_1115 (size=5958) 2024-11-23T18:50:00,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741936_1115 (size=5958) 2024-11-23T18:50:00,263 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/b/876e96b9fc864a9c95c1ebdf3243ebea 2024-11-23T18:50:00,281 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/c/0848cde691b24bdfb90bd9944071c529 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732387800139/Put/seqid=0 2024-11-23T18:50:00,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741937_1116 (size=5958) 2024-11-23T18:50:00,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741937_1116 (size=5958) 2024-11-23T18:50:00,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741937_1116 (size=5958) 2024-11-23T18:50:00,287 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/c/0848cde691b24bdfb90bd9944071c529 2024-11-23T18:50:00,292 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/b/876e96b9fc864a9c95c1ebdf3243ebea as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/b/876e96b9fc864a9c95c1ebdf3243ebea 2024-11-23T18:50:00,296 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/b/876e96b9fc864a9c95c1ebdf3243ebea, entries=10, sequenceid=35, filesize=5.8 K 2024-11-23T18:50:00,297 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/c/0848cde691b24bdfb90bd9944071c529 as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/c/0848cde691b24bdfb90bd9944071c529 2024-11-23T18:50:00,302 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/c/0848cde691b24bdfb90bd9944071c529, entries=10, sequenceid=35, filesize=5.8 K 2024-11-23T18:50:00,302 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 603528e8a0ef8db307680413e0287af5 in 60ms, sequenceid=35, compaction requested=false; wal=null 2024-11-23T18:50:00,302 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000035 2024-11-23T18:50:00,303 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,303 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,304 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:50:00,305 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:00,307 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-23T18:50:00,308 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 603528e8a0ef8db307680413e0287af5; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70959467, jitterRate=0.05737845599651337}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:50:00,308 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 603528e8a0ef8db307680413e0287af5: Writing region info on filesystem at 1732387800205Initializing all the Stores at 1732387800205Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387800205Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387800206 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387800206Obtaining lock to block concurrent updates at 1732387800242 (+36 ms)Preparing flush snapshotting stores in 603528e8a0ef8db307680413e0287af5 at 1732387800242Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732387800242Flushing stores of testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. at 1732387800242Flushing 603528e8a0ef8db307680413e0287af5/b: creating writer at 1732387800242Flushing 603528e8a0ef8db307680413e0287af5/b: appending metadata at 1732387800257 (+15 ms)Flushing 603528e8a0ef8db307680413e0287af5/b: closing flushed file at 1732387800257Flushing 603528e8a0ef8db307680413e0287af5/c: creating writer at 1732387800267 (+10 ms)Flushing 603528e8a0ef8db307680413e0287af5/c: appending metadata at 1732387800280 (+13 ms)Flushing 603528e8a0ef8db307680413e0287af5/c: closing flushed file at 1732387800280Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e318036: reopening flushed file at 1732387800291 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26471002: reopening flushed file at 1732387800297 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 603528e8a0ef8db307680413e0287af5 in 60ms, sequenceid=35, compaction requested=false; wal=null at 1732387800302 (+5 ms)Cleaning up temporary data from old regions at 1732387800303 (+1 ms)Region opened successfully at 1732387800308 (+5 ms) 2024-11-23T18:50:00,384 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187, size=0 (0bytes) 2024-11-23T18:50:00,384 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187 might be still open, length is 0 2024-11-23T18:50:00,384 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187 2024-11-23T18:50:00,385 WARN [IPC Server handler 1 on default port 43835 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-23T18:50:00,385 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187 after 1ms 2024-11-23T18:50:02,320 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:48512 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:37657:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48512 dst: /127.0.0.1:37657 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37657 remote=/127.0.0.1:48512]. Total timeout mills is 60000, 58017 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:50:02,321 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:46492 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:40877:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46492 dst: /127.0.0.1:40877 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:50:02,321 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748881198_22 at /127.0.0.1:52978 [Receiving block BP-995492633-172.17.0.2-1732387759654:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:43523:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52978 dst: /127.0.0.1:43523 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:50:02,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741935_1117 (size=2936) 2024-11-23T18:50:02,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741935_1117 (size=2936) 2024-11-23T18:50:02,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741935_1117 (size=2936) 2024-11-23T18:50:04,386 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187 after 4002ms 2024-11-23T18:50:04,455 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:50:04,456 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187 took 4072ms 2024-11-23T18:50:04,460 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187; continuing. 2024-11-23T18:50:04,460 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187 so closing down 2024-11-23T18:50:04,460 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:50:04,461 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732387800187.temp 2024-11-23T18:50:04,462 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000037-wal.1732387800187.temp 2024-11-23T18:50:04,462 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:50:04,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741938_1118 (size=2944) 2024-11-23T18:50:04,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741938_1118 (size=2944) 2024-11-23T18:50:04,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741938_1118 (size=2944) 2024-11-23T18:50:04,468 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000037-wal.1732387800187.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-23T18:50:04,469 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000037-wal.1732387800187.temp to hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000066 2024-11-23T18:50:04,469 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187, size=0, length=0, corrupted=false, cancelled=false 2024-11-23T18:50:04,469 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187, journal: Splitting hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187, size=0 (0bytes) at 1732387800384Finishing writing output for hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187 so closing down at 1732387804460 (+4076 ms)Creating recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000037-wal.1732387800187.temp at 1732387804462 (+2 ms)3 split writer threads finished at 1732387804462Closed recovered edits writer path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000037-wal.1732387800187.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732387804468 (+6 ms)Rename recovered edits hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000037-wal.1732387800187.temp to hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000066 at 1732387804469 (+1 ms)Processed 30 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187, size=0, length=0, corrupted=false, cancelled=false at 1732387804469 2024-11-23T18:50:04,471 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187 to hdfs://localhost:43835/hbase/oldWALs/wal.1732387800187 2024-11-23T18:50:04,471 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000066 2024-11-23T18:50:04,471 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:50:04,473 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43835/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008, archiveDir=hdfs://localhost:43835/hbase/oldWALs, maxLogs=32 2024-11-23T18:50:04,486 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387804473, exclude list is [], retry=0 2024-11-23T18:50:04,488 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43523,DS-bf15d068-7bb8-4a55-956d-4f5d8fe6bf18,DISK] 2024-11-23T18:50:04,488 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40877,DS-99681bba-29f3-46cb-9183-ad1978fc6996,DISK] 2024-11-23T18:50:04,488 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37657,DS-4bd7ba11-1981-4abe-8a12-566437cf080c,DISK] 2024-11-23T18:50:04,490 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387804473 2024-11-23T18:50:04,490 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43381:43381),(127.0.0.1/127.0.0.1:39619:39619),(127.0.0.1/127.0.0.1:37451:37451)] 2024-11-23T18:50:04,490 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:50:04,492 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:04,492 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 603528e8a0ef8db307680413e0287af5 columnFamilyName a 2024-11-23T18:50:04,492 DEBUG [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:04,497 DEBUG [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/a/8660d36b7ca04dbda4addb6b66aac6fb 2024-11-23T18:50:04,497 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(327): Store=603528e8a0ef8db307680413e0287af5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:50:04,497 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:04,498 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 603528e8a0ef8db307680413e0287af5 columnFamilyName b 2024-11-23T18:50:04,498 DEBUG [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:04,502 DEBUG [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/b/876e96b9fc864a9c95c1ebdf3243ebea 2024-11-23T18:50:04,502 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(327): Store=603528e8a0ef8db307680413e0287af5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:50:04,502 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:04,503 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 603528e8a0ef8db307680413e0287af5 columnFamilyName c 2024-11-23T18:50:04,503 DEBUG [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:04,507 DEBUG [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/c/0848cde691b24bdfb90bd9944071c529 2024-11-23T18:50:04,507 INFO [StoreOpener-603528e8a0ef8db307680413e0287af5-1 {}] regionserver.HStore(327): Store=603528e8a0ef8db307680413e0287af5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:50:04,507 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:04,508 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:04,509 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:04,510 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000066 2024-11-23T18:50:04,511 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-23T18:50:04,516 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000066 2024-11-23T18:50:04,516 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 603528e8a0ef8db307680413e0287af5 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-23T18:50:04,530 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/a/555bb45871004922908643df373cd7d1 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732387800314/Put/seqid=0 2024-11-23T18:50:04,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741940_1120 (size=5958) 2024-11-23T18:50:04,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741940_1120 (size=5958) 2024-11-23T18:50:04,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741940_1120 (size=5958) 2024-11-23T18:50:04,537 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/a/555bb45871004922908643df373cd7d1 2024-11-23T18:50:04,554 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/b/e1760d5386c844f8914b7344193ee9b2 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732387800321/Put/seqid=0 2024-11-23T18:50:04,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741941_1121 (size=5958) 2024-11-23T18:50:04,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741941_1121 (size=5958) 2024-11-23T18:50:04,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741941_1121 (size=5958) 2024-11-23T18:50:04,560 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/b/e1760d5386c844f8914b7344193ee9b2 2024-11-23T18:50:04,577 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/c/9faa3e0ebb8044b9b3f91be231d690f7 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732387800328/Put/seqid=0 2024-11-23T18:50:04,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741942_1122 (size=5958) 2024-11-23T18:50:04,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741942_1122 (size=5958) 2024-11-23T18:50:04,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741942_1122 (size=5958) 2024-11-23T18:50:04,582 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/c/9faa3e0ebb8044b9b3f91be231d690f7 2024-11-23T18:50:04,587 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/a/555bb45871004922908643df373cd7d1 as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/a/555bb45871004922908643df373cd7d1 2024-11-23T18:50:04,591 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/a/555bb45871004922908643df373cd7d1, entries=10, sequenceid=66, filesize=5.8 K 2024-11-23T18:50:04,592 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/b/e1760d5386c844f8914b7344193ee9b2 as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/b/e1760d5386c844f8914b7344193ee9b2 2024-11-23T18:50:04,597 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/b/e1760d5386c844f8914b7344193ee9b2, entries=10, sequenceid=66, filesize=5.8 K 2024-11-23T18:50:04,598 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-23T18:50:04,598 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-23T18:50:04,598 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/.tmp/c/9faa3e0ebb8044b9b3f91be231d690f7 as hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/c/9faa3e0ebb8044b9b3f91be231d690f7 2024-11-23T18:50:04,599 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-23T18:50:04,599 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-23T18:50:04,603 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/c/9faa3e0ebb8044b9b3f91be231d690f7, entries=10, sequenceid=66, filesize=5.8 K 2024-11-23T18:50:04,604 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 603528e8a0ef8db307680413e0287af5 in 87ms, sequenceid=66, compaction requested=false; wal=null 2024-11-23T18:50:04,604 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/0000000000000000066 2024-11-23T18:50:04,605 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:04,605 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:04,606 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:50:04,607 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 603528e8a0ef8db307680413e0287af5 2024-11-23T18:50:04,609 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/hbase/data/default/testReplayEditsWrittenViaHRegion/603528e8a0ef8db307680413e0287af5/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-23T18:50:04,610 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 603528e8a0ef8db307680413e0287af5; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61220008, jitterRate=-0.08775079250335693}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:50:04,610 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 603528e8a0ef8db307680413e0287af5: Writing region info on filesystem at 1732387804491Initializing all the Stores at 1732387804491Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387804491Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387804492 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387804492Obtaining lock to block concurrent updates at 1732387804516 (+24 ms)Preparing flush snapshotting stores in 603528e8a0ef8db307680413e0287af5 at 1732387804516Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732387804516Flushing stores of testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. at 1732387804516Flushing 603528e8a0ef8db307680413e0287af5/a: creating writer at 1732387804516Flushing 603528e8a0ef8db307680413e0287af5/a: appending metadata at 1732387804530 (+14 ms)Flushing 603528e8a0ef8db307680413e0287af5/a: closing flushed file at 1732387804530Flushing 603528e8a0ef8db307680413e0287af5/b: creating writer at 1732387804540 (+10 ms)Flushing 603528e8a0ef8db307680413e0287af5/b: appending metadata at 1732387804553 (+13 ms)Flushing 603528e8a0ef8db307680413e0287af5/b: closing flushed file at 1732387804553Flushing 603528e8a0ef8db307680413e0287af5/c: creating writer at 1732387804563 (+10 ms)Flushing 603528e8a0ef8db307680413e0287af5/c: appending metadata at 1732387804576 (+13 ms)Flushing 603528e8a0ef8db307680413e0287af5/c: closing flushed file at 1732387804576Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@335eaea2: reopening flushed file at 1732387804586 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56ad6dc4: reopening flushed file at 1732387804591 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c603302: reopening flushed file at 1732387804597 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 603528e8a0ef8db307680413e0287af5 in 87ms, sequenceid=66, compaction requested=false; wal=null at 1732387804604 (+7 ms)Cleaning up temporary data from old regions at 1732387804605 (+1 ms)Region opened successfully at 1732387804610 (+5 ms) 2024-11-23T18:50:04,620 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 603528e8a0ef8db307680413e0287af5, disabling compactions & flushes 2024-11-23T18:50:04,620 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. 2024-11-23T18:50:04,620 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. 2024-11-23T18:50:04,620 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. after waiting 0 ms 2024-11-23T18:50:04,620 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. 2024-11-23T18:50:04,621 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732387800009.603528e8a0ef8db307680413e0287af5. 2024-11-23T18:50:04,621 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 603528e8a0ef8db307680413e0287af5: Waiting for close lock at 1732387804620Disabling compacts and flushes for region at 1732387804620Disabling writes for close at 1732387804620Writing region close event to WAL at 1732387804621 (+1 ms)Closed at 1732387804621 2024-11-23T18:50:04,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741939_1119 (size=95) 2024-11-23T18:50:04,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741939_1119 (size=95) 2024-11-23T18:50:04,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741939_1119 (size=95) 2024-11-23T18:50:04,625 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:50:04,626 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732387804473) 2024-11-23T18:50:04,640 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=443 (was 437) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:43835 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_805588319_22 at /127.0.0.1:44798 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:43835 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_805588319_22 at /127.0.0.1:34394 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_805588319_22 at /127.0.0.1:54610 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1408 (was 1342) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=213 (was 232), ProcessCount=11 (was 11), AvailableMemoryMB=8568 (was 8600) 2024-11-23T18:50:04,640 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1408 is superior to 1024 2024-11-23T18:50:04,640 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T18:50:04,640 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T18:50:04,640 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:50:04,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:50:04,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:50:04,641 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T18:50:04,641 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T18:50:04,641 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1885054661, stopped=false 2024-11-23T18:50:04,642 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2eda60943ea1,37117,1732387764258 2024-11-23T18:50:04,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T18:50:04,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T18:50:04,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:50:04,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:50:04,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T18:50:04,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:50:04,776 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T18:50:04,778 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T18:50:04,778 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:50:04,778 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:50:04,778 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:50:04,778 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:50:04,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:50:04,779 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer(878): Closing user regions 2024-11-23T18:50:04,779 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2eda60943ea1,43797,1732387765099' ***** 2024-11-23T18:50:04,779 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T18:50:04,779 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2eda60943ea1,34841,1732387765147' ***** 2024-11-23T18:50:04,779 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T18:50:04,779 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer(3091): Received CLOSE for aafadebc54e468784234072b47bbd24c 2024-11-23T18:50:04,779 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T18:50:04,779 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aafadebc54e468784234072b47bbd24c, disabling compactions & flushes 2024-11-23T18:50:04,780 INFO [RS:2;2eda60943ea1:34841 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T18:50:04,780 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T18:50:04,780 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:50:04,780 INFO [RS:2;2eda60943ea1:34841 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T18:50:04,780 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:50:04,780 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. after waiting 0 ms 2024-11-23T18:50:04,780 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.HRegionServer(959): stopping server 2eda60943ea1,34841,1732387765147 2024-11-23T18:50:04,780 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:50:04,780 INFO [RS:2;2eda60943ea1:34841 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:50:04,780 INFO [RS:2;2eda60943ea1:34841 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;2eda60943ea1:34841. 2024-11-23T18:50:04,780 DEBUG [RS:2;2eda60943ea1:34841 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:50:04,780 DEBUG [RS:2;2eda60943ea1:34841 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:50:04,780 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.HRegionServer(976): stopping server 2eda60943ea1,34841,1732387765147; all regions closed. 2024-11-23T18:50:04,781 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T18:50:04,782 INFO [RS:1;2eda60943ea1:43797 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T18:50:04,782 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T18:50:04,782 INFO [RS:1;2eda60943ea1:43797 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T18:50:04,782 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer(959): stopping server 2eda60943ea1,43797,1732387765099 2024-11-23T18:50:04,782 INFO [RS:1;2eda60943ea1:43797 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:50:04,782 INFO [RS:1;2eda60943ea1:43797 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2eda60943ea1:43797. 2024-11-23T18:50:04,782 DEBUG [RS:1;2eda60943ea1:43797 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:50:04,782 DEBUG [RS:1;2eda60943ea1:43797 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:50:04,782 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T18:50:04,782 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T18:50:04,782 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T18:50:04,782 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T18:50:04,783 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-23T18:50:04,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741834_1010 (size=95) 2024-11-23T18:50:04,783 DEBUG [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, aafadebc54e468784234072b47bbd24c=testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c.} 2024-11-23T18:50:04,783 DEBUG [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, aafadebc54e468784234072b47bbd24c 2024-11-23T18:50:04,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741834_1010 (size=95) 2024-11-23T18:50:04,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741834_1010 (size=95) 2024-11-23T18:50:04,784 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T18:50:04,784 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T18:50:04,784 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T18:50:04,784 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T18:50:04,784 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T18:50:04,784 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-23T18:50:04,785 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/aafadebc54e468784234072b47bbd24c/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-23T18:50:04,786 DEBUG [RS:2;2eda60943ea1:34841 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/oldWALs 2024-11-23T18:50:04,786 INFO [RS:2;2eda60943ea1:34841 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2eda60943ea1%2C34841%2C1732387765147:(num 1732387766690) 2024-11-23T18:50:04,786 DEBUG [RS:2;2eda60943ea1:34841 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:50:04,786 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:50:04,786 INFO [RS:2;2eda60943ea1:34841 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:50:04,786 INFO [RS:2;2eda60943ea1:34841 {}] hbase.ChoreService(370): Chore service for: regionserver/2eda60943ea1:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T18:50:04,786 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:50:04,786 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T18:50:04,786 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aafadebc54e468784234072b47bbd24c: Waiting for close lock at 1732387804779Running coprocessor pre-close hooks at 1732387804779Disabling compacts and flushes for region at 1732387804779Disabling writes for close at 1732387804780 (+1 ms)Writing region close event to WAL at 1732387804781 (+1 ms)Running coprocessor post-close hooks at 1732387804786 (+5 ms)Closed at 1732387804786 2024-11-23T18:50:04,786 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T18:50:04,786 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T18:50:04,786 INFO [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:50:04,786 INFO [RS:2;2eda60943ea1:34841 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:50:04,786 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c. 2024-11-23T18:50:04,786 INFO [RS:2;2eda60943ea1:34841 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34841 2024-11-23T18:50:04,789 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/.tmp/info/fd91692dd4cf4239b066f0933627d349 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732387779962.aafadebc54e468784234072b47bbd24c./info:regioninfo/1732387784381/Put/seqid=0 2024-11-23T18:50:04,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741943_1123 (size=6778) 2024-11-23T18:50:04,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741943_1123 (size=6778) 2024-11-23T18:50:04,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741943_1123 (size=6778) 2024-11-23T18:50:04,797 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/.tmp/info/fd91692dd4cf4239b066f0933627d349 2024-11-23T18:50:04,803 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/.tmp/info/fd91692dd4cf4239b066f0933627d349 as hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/info/fd91692dd4cf4239b066f0933627d349 2024-11-23T18:50:04,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:50:04,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2eda60943ea1,34841,1732387765147 2024-11-23T18:50:04,804 INFO [RS:2;2eda60943ea1:34841 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:50:04,807 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/info/fd91692dd4cf4239b066f0933627d349, entries=8, sequenceid=23, filesize=6.6 K 2024-11-23T18:50:04,808 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false 2024-11-23T18:50:04,813 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-23T18:50:04,814 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T18:50:04,814 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T18:50:04,814 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732387804784Running coprocessor pre-close hooks at 1732387804784Disabling compacts and flushes for region at 1732387804784Disabling writes for close at 1732387804784Obtaining lock to block concurrent updates at 1732387804784Preparing flush snapshotting stores in 1588230740 at 1732387804784Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1732387804784Flushing stores of hbase:meta,,1.1588230740 at 1732387804785 (+1 ms)Flushing 1588230740/info: creating writer at 1732387804785Flushing 1588230740/info: appending metadata at 1732387804789 (+4 ms)Flushing 1588230740/info: closing flushed file at 1732387804789Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65259e5: reopening flushed file at 1732387804802 (+13 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false at 1732387804808 (+6 ms)Writing region close event to WAL at 1732387804810 (+2 ms)Running coprocessor post-close hooks at 1732387804813 (+3 ms)Closed at 1732387804814 (+1 ms) 2024-11-23T18:50:04,814 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T18:50:04,814 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2eda60943ea1,34841,1732387765147] 2024-11-23T18:50:04,825 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2eda60943ea1,34841,1732387765147 already deleted, retry=false 2024-11-23T18:50:04,825 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2eda60943ea1,34841,1732387765147 expired; onlineServers=1 2024-11-23T18:50:04,875 INFO [regionserver/2eda60943ea1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:50:04,878 INFO [regionserver/2eda60943ea1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:50:04,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:50:04,915 INFO [RS:2;2eda60943ea1:34841 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:50:04,915 INFO [RS:2;2eda60943ea1:34841 {}] regionserver.HRegionServer(1031): Exiting; stopping=2eda60943ea1,34841,1732387765147; zookeeper connection closed. 2024-11-23T18:50:04,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34841-0x101690770ba0003, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:50:04,915 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@56c77842 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@56c77842 2024-11-23T18:50:04,984 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer(976): stopping server 2eda60943ea1,43797,1732387765099; all regions closed. 2024-11-23T18:50:04,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741893_1071 (size=1673) 2024-11-23T18:50:04,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741893_1071 (size=1673) 2024-11-23T18:50:04,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741893_1071 (size=1673) 2024-11-23T18:50:04,992 DEBUG [RS:1;2eda60943ea1:43797 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/oldWALs 2024-11-23T18:50:04,992 INFO [RS:1;2eda60943ea1:43797 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2eda60943ea1%2C43797%2C1732387765099.meta:.meta(num 1732387783544) 2024-11-23T18:50:04,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741835_1011 (size=2186) 2024-11-23T18:50:04,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741835_1011 (size=2186) 2024-11-23T18:50:04,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741835_1011 (size=2186) 2024-11-23T18:50:04,997 DEBUG [RS:1;2eda60943ea1:43797 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/oldWALs 2024-11-23T18:50:04,997 INFO [RS:1;2eda60943ea1:43797 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2eda60943ea1%2C43797%2C1732387765099:(num 1732387766693) 2024-11-23T18:50:04,997 DEBUG [RS:1;2eda60943ea1:43797 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:50:04,997 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:50:04,998 INFO [RS:1;2eda60943ea1:43797 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:50:04,998 INFO [RS:1;2eda60943ea1:43797 {}] hbase.ChoreService(370): Chore service for: regionserver/2eda60943ea1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T18:50:04,998 INFO [RS:1;2eda60943ea1:43797 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:50:04,998 INFO [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:50:04,998 INFO [RS:1;2eda60943ea1:43797 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43797 2024-11-23T18:50:05,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:50:05,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2eda60943ea1,43797,1732387765099 2024-11-23T18:50:05,004 INFO [RS:1;2eda60943ea1:43797 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:50:05,014 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2eda60943ea1,43797,1732387765099] 2024-11-23T18:50:05,025 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2eda60943ea1,43797,1732387765099 already deleted, retry=false 2024-11-23T18:50:05,025 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2eda60943ea1,43797,1732387765099 expired; onlineServers=0 2024-11-23T18:50:05,025 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2eda60943ea1,37117,1732387764258' ***** 2024-11-23T18:50:05,025 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T18:50:05,026 INFO [M:0;2eda60943ea1:37117 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:50:05,026 INFO [M:0;2eda60943ea1:37117 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:50:05,026 DEBUG [M:0;2eda60943ea1:37117 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T18:50:05,026 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T18:50:05,026 DEBUG [M:0;2eda60943ea1:37117 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T18:50:05,026 DEBUG [master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387766296 {}] cleaner.HFileCleaner(306): Exit Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387766296,5,FailOnTimeoutGroup] 2024-11-23T18:50:05,026 DEBUG [master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387766295 {}] cleaner.HFileCleaner(306): Exit Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387766295,5,FailOnTimeoutGroup] 2024-11-23T18:50:05,027 INFO [M:0;2eda60943ea1:37117 {}] hbase.ChoreService(370): Chore service for: master/2eda60943ea1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T18:50:05,027 INFO [M:0;2eda60943ea1:37117 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:50:05,027 DEBUG [M:0;2eda60943ea1:37117 {}] master.HMaster(1795): Stopping service threads 2024-11-23T18:50:05,027 INFO [M:0;2eda60943ea1:37117 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T18:50:05,027 INFO [M:0;2eda60943ea1:37117 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T18:50:05,028 INFO [M:0;2eda60943ea1:37117 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T18:50:05,028 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T18:50:05,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T18:50:05,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:50:05,036 DEBUG [M:0;2eda60943ea1:37117 {}] zookeeper.ZKUtil(347): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T18:50:05,036 WARN [M:0;2eda60943ea1:37117 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T18:50:05,037 INFO [M:0;2eda60943ea1:37117 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/.lastflushedseqids 2024-11-23T18:50:05,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741944_1124 (size=119) 2024-11-23T18:50:05,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741944_1124 (size=119) 2024-11-23T18:50:05,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741944_1124 (size=119) 2024-11-23T18:50:05,052 INFO [M:0;2eda60943ea1:37117 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T18:50:05,052 INFO [M:0;2eda60943ea1:37117 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T18:50:05,053 DEBUG [M:0;2eda60943ea1:37117 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T18:50:05,053 INFO [M:0;2eda60943ea1:37117 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:50:05,053 DEBUG [M:0;2eda60943ea1:37117 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:50:05,053 DEBUG [M:0;2eda60943ea1:37117 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T18:50:05,053 DEBUG [M:0;2eda60943ea1:37117 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:50:05,053 INFO [M:0;2eda60943ea1:37117 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-11-23T18:50:05,070 DEBUG [M:0;2eda60943ea1:37117 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/68587b63ac884bb0b7da01134f1dcb23 is 82, key is hbase:meta,,1/info:regioninfo/1732387783703/Put/seqid=0 2024-11-23T18:50:05,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741945_1125 (size=6063) 2024-11-23T18:50:05,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741945_1125 (size=6063) 2024-11-23T18:50:05,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741945_1125 (size=6063) 2024-11-23T18:50:05,076 INFO [M:0;2eda60943ea1:37117 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/68587b63ac884bb0b7da01134f1dcb23 2024-11-23T18:50:05,096 DEBUG [M:0;2eda60943ea1:37117 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a3433f583e164f7cb64c771ba31c9e0d is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732387780399/Put/seqid=0 2024-11-23T18:50:05,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741946_1126 (size=7907) 2024-11-23T18:50:05,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741946_1126 (size=7907) 2024-11-23T18:50:05,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741946_1126 (size=7907) 2024-11-23T18:50:05,104 INFO [M:0;2eda60943ea1:37117 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a3433f583e164f7cb64c771ba31c9e0d 2024-11-23T18:50:05,108 INFO [M:0;2eda60943ea1:37117 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a3433f583e164f7cb64c771ba31c9e0d 2024-11-23T18:50:05,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:50:05,114 INFO [RS:1;2eda60943ea1:43797 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:50:05,114 INFO [RS:1;2eda60943ea1:43797 {}] regionserver.HRegionServer(1031): Exiting; stopping=2eda60943ea1,43797,1732387765099; zookeeper connection closed. 2024-11-23T18:50:05,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43797-0x101690770ba0002, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:50:05,115 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@11ebcd3a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@11ebcd3a 2024-11-23T18:50:05,115 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-23T18:50:05,123 DEBUG [M:0;2eda60943ea1:37117 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/10079e8ba91f44c194fb07b112ff35ac is 69, key is 2eda60943ea1,34841,1732387765147/rs:state/1732387766305/Put/seqid=0 2024-11-23T18:50:05,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741947_1127 (size=5440) 2024-11-23T18:50:05,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741947_1127 (size=5440) 2024-11-23T18:50:05,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741947_1127 (size=5440) 2024-11-23T18:50:05,129 INFO [M:0;2eda60943ea1:37117 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/10079e8ba91f44c194fb07b112ff35ac 2024-11-23T18:50:05,133 INFO [M:0;2eda60943ea1:37117 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 10079e8ba91f44c194fb07b112ff35ac 2024-11-23T18:50:05,134 DEBUG [M:0;2eda60943ea1:37117 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/68587b63ac884bb0b7da01134f1dcb23 as hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/68587b63ac884bb0b7da01134f1dcb23 2024-11-23T18:50:05,138 INFO [M:0;2eda60943ea1:37117 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/68587b63ac884bb0b7da01134f1dcb23, entries=14, sequenceid=207, filesize=5.9 K 2024-11-23T18:50:05,140 DEBUG [M:0;2eda60943ea1:37117 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a3433f583e164f7cb64c771ba31c9e0d as hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a3433f583e164f7cb64c771ba31c9e0d 2024-11-23T18:50:05,144 INFO [M:0;2eda60943ea1:37117 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a3433f583e164f7cb64c771ba31c9e0d 2024-11-23T18:50:05,144 INFO [M:0;2eda60943ea1:37117 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a3433f583e164f7cb64c771ba31c9e0d, entries=21, sequenceid=207, filesize=7.7 K 2024-11-23T18:50:05,145 DEBUG [M:0;2eda60943ea1:37117 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/10079e8ba91f44c194fb07b112ff35ac as hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/10079e8ba91f44c194fb07b112ff35ac 2024-11-23T18:50:05,150 INFO [M:0;2eda60943ea1:37117 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 10079e8ba91f44c194fb07b112ff35ac 2024-11-23T18:50:05,150 INFO [M:0;2eda60943ea1:37117 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43835/user/jenkins/test-data/ec25ca52-863e-f684-bd04-44f389cc47fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/10079e8ba91f44c194fb07b112ff35ac, entries=3, sequenceid=207, filesize=5.3 K 2024-11-23T18:50:05,151 INFO [M:0;2eda60943ea1:37117 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 98ms, sequenceid=207, compaction requested=false 2024-11-23T18:50:05,152 INFO [M:0;2eda60943ea1:37117 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:50:05,152 DEBUG [M:0;2eda60943ea1:37117 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732387805053Disabling compacts and flushes for region at 1732387805053Disabling writes for close at 1732387805053Obtaining lock to block concurrent updates at 1732387805053Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732387805053Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1732387805053Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732387805054 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732387805054Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732387805070 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732387805070Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732387805080 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732387805095 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732387805095Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732387805108 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732387805122 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732387805122Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@173591fe: reopening flushed file at 1732387805133 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@781ec7a6: reopening flushed file at 1732387805139 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fa884a: reopening flushed file at 1732387805144 (+5 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 98ms, sequenceid=207, compaction requested=false at 1732387805151 (+7 ms)Writing region close event to WAL at 1732387805152 (+1 ms)Closed at 1732387805152 2024-11-23T18:50:05,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40877 is added to blk_1073741830_1006 (size=69339) 2024-11-23T18:50:05,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43523 is added to blk_1073741830_1006 (size=69339) 2024-11-23T18:50:05,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37657 is added to blk_1073741830_1006 (size=69339) 2024-11-23T18:50:05,155 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:50:05,155 INFO [M:0;2eda60943ea1:37117 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T18:50:05,155 INFO [M:0;2eda60943ea1:37117 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37117 2024-11-23T18:50:05,155 INFO [M:0;2eda60943ea1:37117 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:50:05,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:50:05,262 INFO [M:0;2eda60943ea1:37117 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:50:05,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37117-0x101690770ba0000, quorum=127.0.0.1:57018, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:50:05,272 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387794261 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387794261 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1748881198_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387784911/wal.1732387794261 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1748881198_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-23T18:50:05,276 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784759 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784759 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1748881198_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387784488/wal.1732387784759 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1748881198_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-23T18:50:05,277 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387775534/wal.1732387775664 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:50:05,278 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387795072 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387795072 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1748881198_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732387794451/wal.1732387795072 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1748881198_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-23T18:50:05,279 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387795217/wal.1732387795679 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:50:05,279 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387800008/wal.1732387800187 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:50:05,279 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387768341/wal.1732387768408 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:50:05,281 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal.1732387768203 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal.1732387768203 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1748881198_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387768032/wal.1732387768203 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1748881198_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-23T18:50:05,283 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal.1732387767805 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal.1732387767805 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1748881198_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387767571/wal.1732387767805 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1748881198_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-23T18:50:05,288 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@35f1150e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:50:05,290 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:50:05,290 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:50:05,291 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:50:05,291 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/hadoop.log.dir/,STOPPED} 2024-11-23T18:50:05,293 WARN [BP-995492633-172.17.0.2-1732387759654 heartbeating to localhost/127.0.0.1:43835 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:50:05,293 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:50:05,293 WARN [BP-995492633-172.17.0.2-1732387759654 heartbeating to localhost/127.0.0.1:43835 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-995492633-172.17.0.2-1732387759654 (Datanode Uuid 3e540eaa-9a8b-4e0d-ba02-a2f0c9380641) service to localhost/127.0.0.1:43835 2024-11-23T18:50:05,293 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:50:05,294 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/data/data5/current/BP-995492633-172.17.0.2-1732387759654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:50:05,295 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/data/data6/current/BP-995492633-172.17.0.2-1732387759654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:50:05,295 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:50:05,297 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bd427b8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:50:05,297 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:50:05,297 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:50:05,297 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:50:05,297 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/hadoop.log.dir/,STOPPED} 2024-11-23T18:50:05,298 WARN [BP-995492633-172.17.0.2-1732387759654 heartbeating to localhost/127.0.0.1:43835 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:50:05,298 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:50:05,298 WARN [BP-995492633-172.17.0.2-1732387759654 heartbeating to localhost/127.0.0.1:43835 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-995492633-172.17.0.2-1732387759654 (Datanode Uuid 717b7c20-a1ca-4fc2-903e-6487ea215d67) service to localhost/127.0.0.1:43835 2024-11-23T18:50:05,298 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:50:05,299 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/data/data3/current/BP-995492633-172.17.0.2-1732387759654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:50:05,299 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/data/data4/current/BP-995492633-172.17.0.2-1732387759654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:50:05,299 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:50:05,305 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@330740de{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:50:05,305 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:50:05,305 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:50:05,306 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:50:05,306 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/hadoop.log.dir/,STOPPED} 2024-11-23T18:50:05,307 WARN [BP-995492633-172.17.0.2-1732387759654 heartbeating to localhost/127.0.0.1:43835 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:50:05,307 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:50:05,307 WARN [BP-995492633-172.17.0.2-1732387759654 heartbeating to localhost/127.0.0.1:43835 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-995492633-172.17.0.2-1732387759654 (Datanode Uuid fd34b4b4-85d3-4d1a-8085-dd5cabccc71e) service to localhost/127.0.0.1:43835 2024-11-23T18:50:05,307 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:50:05,307 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/data/data1/current/BP-995492633-172.17.0.2-1732387759654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:50:05,307 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/cluster_f3968f4e-ed43-8b57-8de5-da4c214c4789/data/data2/current/BP-995492633-172.17.0.2-1732387759654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:50:05,308 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:50:05,313 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3717288f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T18:50:05,313 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:50:05,313 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:50:05,314 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:50:05,314 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0d9fdf29-906d-de14-5d87-d1e4e3bceeef/hadoop.log.dir/,STOPPED} 2024-11-23T18:50:05,322 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T18:50:05,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down