2024-12-09 03:48:08,450 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7857fe2 2024-12-09 03:48:08,465 main DEBUG Took 0.012685 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 03:48:08,465 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 03:48:08,466 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 03:48:08,467 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 03:48:08,468 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:48:08,477 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 03:48:08,493 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:48:08,494 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:48:08,495 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:48:08,495 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:48:08,496 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:48:08,496 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:48:08,497 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:48:08,498 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:48:08,498 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:48:08,498 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:48:08,499 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:48:08,500 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:48:08,500 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:48:08,501 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:48:08,501 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:48:08,501 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:48:08,502 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:48:08,502 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:48:08,503 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:48:08,503 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:48:08,504 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:48:08,504 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:48:08,505 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:48:08,505 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:48:08,506 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:48:08,506 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 03:48:08,507 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:48:08,509 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 03:48:08,511 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 03:48:08,512 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 03:48:08,514 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 03:48:08,514 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 03:48:08,526 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 03:48:08,530 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 03:48:08,532 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 03:48:08,532 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 03:48:08,533 main DEBUG createAppenders(={Console}) 2024-12-09 03:48:08,534 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7857fe2 initialized 2024-12-09 03:48:08,534 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7857fe2 2024-12-09 03:48:08,535 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7857fe2 OK. 2024-12-09 03:48:08,535 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 03:48:08,536 main DEBUG OutputStream closed 2024-12-09 03:48:08,536 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 03:48:08,537 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 03:48:08,537 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@54a7079e OK 2024-12-09 03:48:08,631 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 03:48:08,634 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 03:48:08,635 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 03:48:08,636 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 03:48:08,637 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 03:48:08,637 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 03:48:08,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 03:48:08,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 03:48:08,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 03:48:08,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 03:48:08,640 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 03:48:08,640 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 03:48:08,641 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 03:48:08,641 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 03:48:08,641 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 03:48:08,642 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 03:48:08,642 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 03:48:08,643 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 03:48:08,646 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 03:48:08,646 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19b93fa8) with optional ClassLoader: null 2024-12-09 03:48:08,647 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 03:48:08,648 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19b93fa8] started OK. 2024-12-09T03:48:09,018 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae 2024-12-09 03:48:09,023 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 03:48:09,023 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T03:48:09,037 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-09T03:48:09,046 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-12-09T03:48:09,076 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T03:48:09,139 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T03:48:09,139 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T03:48:09,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T03:48:09,174 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc, deleteOnExit=true 2024-12-09T03:48:09,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T03:48:09,176 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/test.cache.data in system properties and HBase conf 2024-12-09T03:48:09,177 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T03:48:09,177 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/hadoop.log.dir in system properties and HBase conf 2024-12-09T03:48:09,178 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T03:48:09,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T03:48:09,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T03:48:09,282 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T03:48:09,421 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T03:48:09,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T03:48:09,426 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T03:48:09,426 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T03:48:09,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T03:48:09,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T03:48:09,428 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T03:48:09,428 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T03:48:09,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T03:48:09,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T03:48:09,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/nfs.dump.dir in system properties and HBase conf 2024-12-09T03:48:09,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/java.io.tmpdir in system properties and HBase conf 2024-12-09T03:48:09,431 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T03:48:09,431 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T03:48:09,432 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T03:48:10,397 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T03:48:10,484 INFO [Time-limited test {}] log.Log(170): Logging initialized @2906ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T03:48:10,564 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T03:48:10,625 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T03:48:10,647 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T03:48:10,647 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T03:48:10,649 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T03:48:10,669 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T03:48:10,671 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/hadoop.log.dir/,AVAILABLE} 2024-12-09T03:48:10,672 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T03:48:10,868 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/java.io.tmpdir/jetty-localhost-36685-hadoop-hdfs-3_4_1-tests_jar-_-any-1877872976967898926/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T03:48:10,874 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:36685} 2024-12-09T03:48:10,874 INFO [Time-limited test {}] server.Server(415): Started @3296ms 2024-12-09T03:48:11,281 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T03:48:11,288 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T03:48:11,289 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T03:48:11,289 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T03:48:11,289 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T03:48:11,290 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/hadoop.log.dir/,AVAILABLE} 2024-12-09T03:48:11,291 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T03:48:11,411 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/java.io.tmpdir/jetty-localhost-38861-hadoop-hdfs-3_4_1-tests_jar-_-any-12320428935350647156/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T03:48:11,412 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:38861} 2024-12-09T03:48:11,412 INFO [Time-limited test {}] server.Server(415): Started @3834ms 2024-12-09T03:48:11,467 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T03:48:11,585 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T03:48:11,593 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T03:48:11,597 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T03:48:11,597 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T03:48:11,598 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T03:48:11,599 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/hadoop.log.dir/,AVAILABLE} 2024-12-09T03:48:11,600 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T03:48:11,753 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/java.io.tmpdir/jetty-localhost-38007-hadoop-hdfs-3_4_1-tests_jar-_-any-14316854796268786345/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T03:48:11,754 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:38007} 2024-12-09T03:48:11,754 INFO [Time-limited test {}] server.Server(415): Started @4176ms 2024-12-09T03:48:11,757 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T03:48:11,805 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T03:48:11,810 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T03:48:11,814 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T03:48:11,814 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T03:48:11,814 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T03:48:11,816 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/hadoop.log.dir/,AVAILABLE} 2024-12-09T03:48:11,817 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T03:48:11,920 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data2/current/BP-2031904063-172.17.0.2-1733716090120/current, will proceed with Du for space computation calculation, 2024-12-09T03:48:11,920 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data4/current/BP-2031904063-172.17.0.2-1733716090120/current, will proceed with Du for space computation calculation, 2024-12-09T03:48:11,920 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data1/current/BP-2031904063-172.17.0.2-1733716090120/current, will proceed with Du for space computation calculation, 2024-12-09T03:48:11,920 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data3/current/BP-2031904063-172.17.0.2-1733716090120/current, will proceed with Du for space computation calculation, 2024-12-09T03:48:11,943 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/java.io.tmpdir/jetty-localhost-42557-hadoop-hdfs-3_4_1-tests_jar-_-any-4217965005757428055/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T03:48:11,944 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:42557} 2024-12-09T03:48:11,944 INFO [Time-limited test {}] server.Server(415): Started @4366ms 2024-12-09T03:48:11,947 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T03:48:11,965 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T03:48:11,965 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T03:48:12,041 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5262a8eb6e0bc14e with lease ID 0x2a750c6d1eedf58: Processing first storage report for DS-21e3b170-8d5d-42d3-83cd-217896bbed1f from datanode DatanodeRegistration(127.0.0.1:39115, datanodeUuid=40d5c32f-369d-4872-94b6-2f2d7378e3e0, infoPort=38745, infoSecurePort=0, ipcPort=44623, storageInfo=lv=-57;cid=testClusterID;nsid=504844639;c=1733716090120) 2024-12-09T03:48:12,042 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5262a8eb6e0bc14e with lease ID 0x2a750c6d1eedf58: from storage DS-21e3b170-8d5d-42d3-83cd-217896bbed1f node DatanodeRegistration(127.0.0.1:39115, datanodeUuid=40d5c32f-369d-4872-94b6-2f2d7378e3e0, infoPort=38745, infoSecurePort=0, ipcPort=44623, storageInfo=lv=-57;cid=testClusterID;nsid=504844639;c=1733716090120), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-09T03:48:12,042 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcc013369bba5c26 with lease ID 0x2a750c6d1eedf59: Processing first storage report for DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d from datanode DatanodeRegistration(127.0.0.1:36217, datanodeUuid=796afa67-5d59-4e50-bcd5-6fb248bd28eb, infoPort=36521, infoSecurePort=0, ipcPort=45919, storageInfo=lv=-57;cid=testClusterID;nsid=504844639;c=1733716090120) 2024-12-09T03:48:12,042 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcc013369bba5c26 with lease ID 0x2a750c6d1eedf59: from storage DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d node DatanodeRegistration(127.0.0.1:36217, datanodeUuid=796afa67-5d59-4e50-bcd5-6fb248bd28eb, infoPort=36521, infoSecurePort=0, ipcPort=45919, storageInfo=lv=-57;cid=testClusterID;nsid=504844639;c=1733716090120), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T03:48:12,043 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5262a8eb6e0bc14e with lease ID 0x2a750c6d1eedf58: Processing first storage report for DS-781670a3-6b65-4098-a781-3d0e2c5608bf from datanode DatanodeRegistration(127.0.0.1:39115, datanodeUuid=40d5c32f-369d-4872-94b6-2f2d7378e3e0, infoPort=38745, infoSecurePort=0, ipcPort=44623, storageInfo=lv=-57;cid=testClusterID;nsid=504844639;c=1733716090120) 2024-12-09T03:48:12,043 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5262a8eb6e0bc14e with lease ID 0x2a750c6d1eedf58: from storage DS-781670a3-6b65-4098-a781-3d0e2c5608bf node DatanodeRegistration(127.0.0.1:39115, datanodeUuid=40d5c32f-369d-4872-94b6-2f2d7378e3e0, infoPort=38745, infoSecurePort=0, ipcPort=44623, storageInfo=lv=-57;cid=testClusterID;nsid=504844639;c=1733716090120), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T03:48:12,043 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcc013369bba5c26 with lease ID 0x2a750c6d1eedf59: Processing first storage report for DS-5405e5bb-52dd-4391-b9f9-13cf91fe70ac from datanode DatanodeRegistration(127.0.0.1:36217, datanodeUuid=796afa67-5d59-4e50-bcd5-6fb248bd28eb, infoPort=36521, infoSecurePort=0, ipcPort=45919, storageInfo=lv=-57;cid=testClusterID;nsid=504844639;c=1733716090120) 2024-12-09T03:48:12,043 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcc013369bba5c26 with lease ID 0x2a750c6d1eedf59: from storage DS-5405e5bb-52dd-4391-b9f9-13cf91fe70ac node DatanodeRegistration(127.0.0.1:36217, datanodeUuid=796afa67-5d59-4e50-bcd5-6fb248bd28eb, infoPort=36521, infoSecurePort=0, ipcPort=45919, storageInfo=lv=-57;cid=testClusterID;nsid=504844639;c=1733716090120), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T03:48:12,085 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data5/current/BP-2031904063-172.17.0.2-1733716090120/current, will proceed with Du for space computation calculation, 2024-12-09T03:48:12,086 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data6/current/BP-2031904063-172.17.0.2-1733716090120/current, will proceed with Du for space computation calculation, 2024-12-09T03:48:12,116 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T03:48:12,122 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xca8cf6465d427c78 with lease ID 0x2a750c6d1eedf5a: Processing first storage report for DS-08197379-152b-4313-b695-1141a205ab89 from datanode DatanodeRegistration(127.0.0.1:45815, datanodeUuid=a349d05a-2373-4b67-812b-e88082e5940a, infoPort=37281, infoSecurePort=0, ipcPort=34987, storageInfo=lv=-57;cid=testClusterID;nsid=504844639;c=1733716090120) 2024-12-09T03:48:12,122 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xca8cf6465d427c78 with lease ID 0x2a750c6d1eedf5a: from storage DS-08197379-152b-4313-b695-1141a205ab89 node DatanodeRegistration(127.0.0.1:45815, datanodeUuid=a349d05a-2373-4b67-812b-e88082e5940a, infoPort=37281, infoSecurePort=0, ipcPort=34987, storageInfo=lv=-57;cid=testClusterID;nsid=504844639;c=1733716090120), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T03:48:12,122 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xca8cf6465d427c78 with lease ID 0x2a750c6d1eedf5a: Processing first storage report for DS-12b0618b-f818-4bc2-b985-e1ef5c32db0e from datanode DatanodeRegistration(127.0.0.1:45815, datanodeUuid=a349d05a-2373-4b67-812b-e88082e5940a, infoPort=37281, infoSecurePort=0, ipcPort=34987, storageInfo=lv=-57;cid=testClusterID;nsid=504844639;c=1733716090120) 2024-12-09T03:48:12,122 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xca8cf6465d427c78 with lease ID 0x2a750c6d1eedf5a: from storage DS-12b0618b-f818-4bc2-b985-e1ef5c32db0e node DatanodeRegistration(127.0.0.1:45815, datanodeUuid=a349d05a-2373-4b67-812b-e88082e5940a, infoPort=37281, infoSecurePort=0, ipcPort=34987, storageInfo=lv=-57;cid=testClusterID;nsid=504844639;c=1733716090120), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T03:48:12,321 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae 2024-12-09T03:48:12,400 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/zookeeper_0, clientPort=62749, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T03:48:12,411 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62749 2024-12-09T03:48:12,421 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:48:12,424 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:48:12,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741825_1001 (size=7) 2024-12-09T03:48:12,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741825_1001 (size=7) 2024-12-09T03:48:12,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741825_1001 (size=7) 2024-12-09T03:48:13,083 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5 with version=8 2024-12-09T03:48:13,084 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/hbase-staging 2024-12-09T03:48:13,396 INFO [Time-limited test {}] client.ConnectionUtils(128): master/55d0183f16d2:0 server-side Connection retries=45 2024-12-09T03:48:13,407 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T03:48:13,408 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T03:48:13,412 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T03:48:13,413 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T03:48:13,413 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T03:48:13,552 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T03:48:13,614 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T03:48:13,623 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T03:48:13,627 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T03:48:13,656 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 31456 (auto-detected) 2024-12-09T03:48:13,657 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-09T03:48:13,677 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34431 2024-12-09T03:48:13,699 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34431 connecting to ZooKeeper ensemble=127.0.0.1:62749 2024-12-09T03:48:13,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:344310x0, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:48:13,737 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34431-0x1019556106d0000 connected 2024-12-09T03:48:13,770 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:48:13,774 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:48:13,786 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T03:48:13,792 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5, hbase.cluster.distributed=false 2024-12-09T03:48:13,823 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T03:48:13,830 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34431 2024-12-09T03:48:13,831 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34431 2024-12-09T03:48:13,831 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34431 2024-12-09T03:48:13,831 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34431 2024-12-09T03:48:13,832 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34431 2024-12-09T03:48:13,945 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/55d0183f16d2:0 server-side Connection retries=45 2024-12-09T03:48:13,947 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T03:48:13,948 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T03:48:13,948 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T03:48:13,948 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T03:48:13,948 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T03:48:13,951 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T03:48:13,954 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T03:48:13,955 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39859 2024-12-09T03:48:13,958 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39859 connecting to ZooKeeper ensemble=127.0.0.1:62749 2024-12-09T03:48:13,960 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:48:13,966 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:48:13,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:398590x0, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:48:13,981 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:398590x0, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T03:48:13,982 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39859-0x1019556106d0001 connected 2024-12-09T03:48:13,986 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T03:48:13,996 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T03:48:13,999 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T03:48:14,005 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T03:48:14,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39859 2024-12-09T03:48:14,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39859 2024-12-09T03:48:14,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39859 2024-12-09T03:48:14,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39859 2024-12-09T03:48:14,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39859 2024-12-09T03:48:14,038 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/55d0183f16d2:0 server-side Connection retries=45 2024-12-09T03:48:14,039 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T03:48:14,039 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T03:48:14,040 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T03:48:14,040 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T03:48:14,040 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T03:48:14,041 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T03:48:14,041 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T03:48:14,042 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45651 2024-12-09T03:48:14,045 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45651 connecting to ZooKeeper ensemble=127.0.0.1:62749 2024-12-09T03:48:14,046 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:48:14,051 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:48:14,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:456510x0, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:48:14,060 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:456510x0, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T03:48:14,060 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45651-0x1019556106d0002 connected 2024-12-09T03:48:14,060 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T03:48:14,061 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T03:48:14,062 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45651-0x1019556106d0002, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T03:48:14,064 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45651-0x1019556106d0002, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T03:48:14,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45651 2024-12-09T03:48:14,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45651 2024-12-09T03:48:14,066 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45651 2024-12-09T03:48:14,066 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45651 2024-12-09T03:48:14,067 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45651 2024-12-09T03:48:14,084 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/55d0183f16d2:0 server-side Connection retries=45 2024-12-09T03:48:14,085 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T03:48:14,085 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T03:48:14,085 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T03:48:14,085 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T03:48:14,085 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T03:48:14,085 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T03:48:14,085 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T03:48:14,086 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35501 2024-12-09T03:48:14,088 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35501 connecting to ZooKeeper ensemble=127.0.0.1:62749 2024-12-09T03:48:14,089 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:48:14,091 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:48:14,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:355010x0, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:48:14,097 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T03:48:14,097 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35501-0x1019556106d0003 connected 2024-12-09T03:48:14,097 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T03:48:14,098 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T03:48:14,099 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T03:48:14,101 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T03:48:14,104 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35501 2024-12-09T03:48:14,105 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35501 2024-12-09T03:48:14,105 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35501 2024-12-09T03:48:14,106 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35501 2024-12-09T03:48:14,106 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35501 2024-12-09T03:48:14,122 DEBUG [M:0;55d0183f16d2:34431 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;55d0183f16d2:34431 2024-12-09T03:48:14,123 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/55d0183f16d2,34431,1733716093194 2024-12-09T03:48:14,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45651-0x1019556106d0002, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T03:48:14,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T03:48:14,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T03:48:14,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T03:48:14,132 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/55d0183f16d2,34431,1733716093194 2024-12-09T03:48:14,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T03:48:14,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T03:48:14,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45651-0x1019556106d0002, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T03:48:14,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45651-0x1019556106d0002, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,155 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T03:48:14,157 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/55d0183f16d2,34431,1733716093194 from backup master directory 2024-12-09T03:48:14,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/55d0183f16d2,34431,1733716093194 2024-12-09T03:48:14,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T03:48:14,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T03:48:14,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45651-0x1019556106d0002, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T03:48:14,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T03:48:14,163 WARN [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T03:48:14,163 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=55d0183f16d2,34431,1733716093194 2024-12-09T03:48:14,165 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T03:48:14,166 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T03:48:14,224 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/hbase.id] with ID: 79d41f04-398a-445f-90b8-79288500355c 2024-12-09T03:48:14,225 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/.tmp/hbase.id 2024-12-09T03:48:14,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741826_1002 (size=42) 2024-12-09T03:48:14,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741826_1002 (size=42) 2024-12-09T03:48:14,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741826_1002 (size=42) 2024-12-09T03:48:14,240 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/.tmp/hbase.id]:[hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/hbase.id] 2024-12-09T03:48:14,284 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:48:14,288 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T03:48:14,308 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-12-09T03:48:14,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45651-0x1019556106d0002, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741827_1003 (size=196) 2024-12-09T03:48:14,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741827_1003 (size=196) 2024-12-09T03:48:14,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741827_1003 (size=196) 2024-12-09T03:48:14,349 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T03:48:14,351 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T03:48:14,357 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:14,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741828_1004 (size=1189) 2024-12-09T03:48:14,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741828_1004 (size=1189) 2024-12-09T03:48:14,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741828_1004 (size=1189) 2024-12-09T03:48:14,410 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store 2024-12-09T03:48:14,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741829_1005 (size=34) 2024-12-09T03:48:14,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741829_1005 (size=34) 2024-12-09T03:48:14,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741829_1005 (size=34) 2024-12-09T03:48:14,439 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T03:48:14,443 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:14,444 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T03:48:14,444 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:48:14,444 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:48:14,445 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T03:48:14,446 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:48:14,446 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:48:14,447 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733716094444Disabling compacts and flushes for region at 1733716094444Disabling writes for close at 1733716094446 (+2 ms)Writing region close event to WAL at 1733716094446Closed at 1733716094446 2024-12-09T03:48:14,449 WARN [master/55d0183f16d2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/.initializing 2024-12-09T03:48:14,449 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/WALs/55d0183f16d2,34431,1733716093194 2024-12-09T03:48:14,457 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:48:14,472 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=55d0183f16d2%2C34431%2C1733716093194, suffix=, logDir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/WALs/55d0183f16d2,34431,1733716093194, archiveDir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/oldWALs, maxLogs=10 2024-12-09T03:48:14,501 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/WALs/55d0183f16d2,34431,1733716093194/55d0183f16d2%2C34431%2C1733716093194.1733716094477, exclude list is [], retry=0 2024-12-09T03:48:14,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:14,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:14,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:14,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:14,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T03:48:14,569 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-12-09T03:48:14,631 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/WALs/55d0183f16d2,34431,1733716093194/55d0183f16d2%2C34431%2C1733716093194.1733716094477 2024-12-09T03:48:14,632 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:36521:36521)] 2024-12-09T03:48:14,632 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:14,633 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:14,636 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:48:14,636 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:48:14,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:48:14,700 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T03:48:14,703 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:14,706 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:48:14,706 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:48:14,709 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T03:48:14,709 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:14,710 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:14,711 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:48:14,713 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T03:48:14,713 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:14,714 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:14,714 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:48:14,717 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T03:48:14,717 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:14,718 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:14,718 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:48:14,722 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:48:14,723 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:48:14,728 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:48:14,729 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:48:14,733 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T03:48:14,737 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:48:14,741 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:48:14,742 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64600259, jitterRate=-0.0373811274766922}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T03:48:14,748 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733716094649Initializing all the Stores at 1733716094651 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733716094652 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716094653 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716094653Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716094653Cleaning up temporary data from old regions at 1733716094729 (+76 ms)Region opened successfully at 1733716094748 (+19 ms) 2024-12-09T03:48:14,750 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T03:48:14,785 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cfc48c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=55d0183f16d2/172.17.0.2:0 2024-12-09T03:48:14,818 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T03:48:14,830 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T03:48:14,830 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T03:48:14,833 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T03:48:14,835 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T03:48:14,840 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-09T03:48:14,840 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T03:48:14,867 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T03:48:14,876 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T03:48:14,878 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T03:48:14,880 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T03:48:14,881 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T03:48:14,883 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T03:48:14,885 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T03:48:14,888 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T03:48:14,891 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T03:48:14,892 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T03:48:14,893 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T03:48:14,911 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T03:48:14,912 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T03:48:14,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T03:48:14,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T03:48:14,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45651-0x1019556106d0002, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T03:48:14,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T03:48:14,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45651-0x1019556106d0002, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,919 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=55d0183f16d2,34431,1733716093194, sessionid=0x1019556106d0000, setting cluster-up flag (Was=false) 2024-12-09T03:48:14,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45651-0x1019556106d0002, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,941 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T03:48:14,943 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=55d0183f16d2,34431,1733716093194 2024-12-09T03:48:14,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45651-0x1019556106d0002, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:14,956 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T03:48:14,957 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=55d0183f16d2,34431,1733716093194 2024-12-09T03:48:14,964 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T03:48:15,011 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.HRegionServer(746): ClusterId : 79d41f04-398a-445f-90b8-79288500355c 2024-12-09T03:48:15,011 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(746): ClusterId : 79d41f04-398a-445f-90b8-79288500355c 2024-12-09T03:48:15,011 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(746): ClusterId : 79d41f04-398a-445f-90b8-79288500355c 2024-12-09T03:48:15,013 DEBUG [RS:0;55d0183f16d2:39859 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T03:48:15,013 DEBUG [RS:2;55d0183f16d2:35501 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T03:48:15,013 DEBUG [RS:1;55d0183f16d2:45651 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T03:48:15,019 DEBUG [RS:1;55d0183f16d2:45651 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T03:48:15,019 DEBUG [RS:2;55d0183f16d2:35501 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T03:48:15,019 DEBUG [RS:0;55d0183f16d2:39859 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T03:48:15,019 DEBUG [RS:1;55d0183f16d2:45651 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T03:48:15,019 DEBUG [RS:2;55d0183f16d2:35501 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T03:48:15,019 DEBUG [RS:0;55d0183f16d2:39859 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T03:48:15,022 DEBUG [RS:1;55d0183f16d2:45651 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T03:48:15,022 DEBUG [RS:2;55d0183f16d2:35501 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T03:48:15,022 DEBUG [RS:0;55d0183f16d2:39859 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T03:48:15,022 DEBUG [RS:1;55d0183f16d2:45651 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13b36502, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=55d0183f16d2/172.17.0.2:0 2024-12-09T03:48:15,022 DEBUG [RS:0;55d0183f16d2:39859 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ac04fd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=55d0183f16d2/172.17.0.2:0 2024-12-09T03:48:15,022 DEBUG [RS:2;55d0183f16d2:35501 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50202ec5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=55d0183f16d2/172.17.0.2:0 2024-12-09T03:48:15,037 DEBUG [RS:2;55d0183f16d2:35501 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;55d0183f16d2:35501 2024-12-09T03:48:15,039 DEBUG [RS:1;55d0183f16d2:45651 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;55d0183f16d2:45651 2024-12-09T03:48:15,041 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T03:48:15,041 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T03:48:15,041 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T03:48:15,041 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T03:48:15,041 DEBUG [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T03:48:15,041 DEBUG [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T03:48:15,042 DEBUG [RS:0;55d0183f16d2:39859 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;55d0183f16d2:39859 2024-12-09T03:48:15,042 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T03:48:15,043 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T03:48:15,043 DEBUG [RS:0;55d0183f16d2:39859 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T03:48:15,044 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(2659): reportForDuty to master=55d0183f16d2,34431,1733716093194 with port=35501, startcode=1733716094084 2024-12-09T03:48:15,044 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.HRegionServer(2659): reportForDuty to master=55d0183f16d2,34431,1733716093194 with port=39859, startcode=1733716093905 2024-12-09T03:48:15,044 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(2659): reportForDuty to master=55d0183f16d2,34431,1733716093194 with port=45651, startcode=1733716094038 2024-12-09T03:48:15,052 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T03:48:15,056 DEBUG [RS:0;55d0183f16d2:39859 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T03:48:15,056 DEBUG [RS:2;55d0183f16d2:35501 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T03:48:15,056 DEBUG [RS:1;55d0183f16d2:45651 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T03:48:15,065 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T03:48:15,075 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T03:48:15,082 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 55d0183f16d2,34431,1733716093194 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T03:48:15,092 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/55d0183f16d2:0, corePoolSize=5, maxPoolSize=5 2024-12-09T03:48:15,092 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/55d0183f16d2:0, corePoolSize=5, maxPoolSize=5 2024-12-09T03:48:15,092 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/55d0183f16d2:0, corePoolSize=5, maxPoolSize=5 2024-12-09T03:48:15,092 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/55d0183f16d2:0, corePoolSize=5, maxPoolSize=5 2024-12-09T03:48:15,093 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/55d0183f16d2:0, corePoolSize=10, maxPoolSize=10 2024-12-09T03:48:15,093 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,093 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59979, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T03:48:15,093 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55861, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T03:48:15,093 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60741, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T03:48:15,093 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/55d0183f16d2:0, corePoolSize=2, maxPoolSize=2 2024-12-09T03:48:15,093 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,097 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733716125097 2024-12-09T03:48:15,098 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T03:48:15,099 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T03:48:15,101 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34431 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T03:48:15,102 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T03:48:15,103 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T03:48:15,103 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T03:48:15,103 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T03:48:15,104 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T03:48:15,104 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T03:48:15,105 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,108 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T03:48:15,109 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T03:48:15,109 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34431 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T03:48:15,109 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T03:48:15,110 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34431 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T03:48:15,110 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:15,111 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T03:48:15,112 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T03:48:15,112 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T03:48:15,114 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/55d0183f16d2:0:becomeActiveMaster-HFileCleaner.large.0-1733716095113,5,FailOnTimeoutGroup] 2024-12-09T03:48:15,118 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/55d0183f16d2:0:becomeActiveMaster-HFileCleaner.small.0-1733716095114,5,FailOnTimeoutGroup] 2024-12-09T03:48:15,118 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,118 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T03:48:15,120 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,120 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741831_1007 (size=1321) 2024-12-09T03:48:15,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741831_1007 (size=1321) 2024-12-09T03:48:15,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741831_1007 (size=1321) 2024-12-09T03:48:15,162 DEBUG [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T03:48:15,162 DEBUG [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T03:48:15,162 DEBUG [RS:0;55d0183f16d2:39859 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T03:48:15,162 WARN [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T03:48:15,162 WARN [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T03:48:15,162 WARN [RS:0;55d0183f16d2:39859 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T03:48:15,163 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T03:48:15,163 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5 2024-12-09T03:48:15,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741832_1008 (size=32) 2024-12-09T03:48:15,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741832_1008 (size=32) 2024-12-09T03:48:15,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741832_1008 (size=32) 2024-12-09T03:48:15,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:15,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T03:48:15,183 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T03:48:15,184 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:15,184 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:48:15,185 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T03:48:15,187 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T03:48:15,187 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:15,188 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:48:15,188 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T03:48:15,190 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T03:48:15,191 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:15,192 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:48:15,192 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T03:48:15,195 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T03:48:15,195 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:15,196 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:48:15,196 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T03:48:15,197 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740 2024-12-09T03:48:15,198 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740 2024-12-09T03:48:15,200 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T03:48:15,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T03:48:15,202 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T03:48:15,204 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T03:48:15,209 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:48:15,210 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73999611, jitterRate=0.10268013179302216}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T03:48:15,212 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733716095179Initializing all the Stores at 1733716095181 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733716095181Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733716095181Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716095181Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733716095181Cleaning up temporary data from old regions at 1733716095201 (+20 ms)Region opened successfully at 1733716095212 (+11 ms) 2024-12-09T03:48:15,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T03:48:15,213 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T03:48:15,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T03:48:15,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T03:48:15,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T03:48:15,214 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T03:48:15,215 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733716095213Disabling compacts and flushes for region at 1733716095213Disabling writes for close at 1733716095213Writing region close event to WAL at 1733716095214 (+1 ms)Closed at 1733716095214 2024-12-09T03:48:15,218 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T03:48:15,218 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T03:48:15,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T03:48:15,235 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T03:48:15,239 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T03:48:15,264 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(2659): reportForDuty to master=55d0183f16d2,34431,1733716093194 with port=45651, startcode=1733716094038 2024-12-09T03:48:15,264 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.HRegionServer(2659): reportForDuty to master=55d0183f16d2,34431,1733716093194 with port=39859, startcode=1733716093905 2024-12-09T03:48:15,264 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(2659): reportForDuty to master=55d0183f16d2,34431,1733716093194 with port=35501, startcode=1733716094084 2024-12-09T03:48:15,266 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34431 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 55d0183f16d2,39859,1733716093905 2024-12-09T03:48:15,269 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34431 {}] master.ServerManager(517): Registering regionserver=55d0183f16d2,39859,1733716093905 2024-12-09T03:48:15,280 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34431 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 55d0183f16d2,35501,1733716094084 2024-12-09T03:48:15,280 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34431 {}] master.ServerManager(517): Registering regionserver=55d0183f16d2,35501,1733716094084 2024-12-09T03:48:15,280 DEBUG [RS:0;55d0183f16d2:39859 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5 2024-12-09T03:48:15,280 DEBUG [RS:0;55d0183f16d2:39859 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35437 2024-12-09T03:48:15,280 DEBUG [RS:0;55d0183f16d2:39859 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T03:48:15,283 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34431 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 55d0183f16d2,45651,1733716094038 2024-12-09T03:48:15,283 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34431 {}] master.ServerManager(517): Registering regionserver=55d0183f16d2,45651,1733716094038 2024-12-09T03:48:15,283 DEBUG [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5 2024-12-09T03:48:15,284 DEBUG [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35437 2024-12-09T03:48:15,284 DEBUG [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T03:48:15,286 DEBUG [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5 2024-12-09T03:48:15,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T03:48:15,287 DEBUG [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35437 2024-12-09T03:48:15,287 DEBUG [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T03:48:15,297 DEBUG [RS:0;55d0183f16d2:39859 {}] zookeeper.ZKUtil(111): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/55d0183f16d2,39859,1733716093905 2024-12-09T03:48:15,297 WARN [RS:0;55d0183f16d2:39859 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T03:48:15,297 INFO [RS:0;55d0183f16d2:39859 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:15,298 DEBUG [RS:0;55d0183f16d2:39859 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,39859,1733716093905 2024-12-09T03:48:15,298 DEBUG [RS:2;55d0183f16d2:35501 {}] zookeeper.ZKUtil(111): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/55d0183f16d2,35501,1733716094084 2024-12-09T03:48:15,298 WARN [RS:2;55d0183f16d2:35501 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T03:48:15,298 INFO [RS:2;55d0183f16d2:35501 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:15,299 DEBUG [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,35501,1733716094084 2024-12-09T03:48:15,299 DEBUG [RS:1;55d0183f16d2:45651 {}] zookeeper.ZKUtil(111): regionserver:45651-0x1019556106d0002, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/55d0183f16d2,45651,1733716094038 2024-12-09T03:48:15,299 WARN [RS:1;55d0183f16d2:45651 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T03:48:15,299 INFO [RS:1;55d0183f16d2:45651 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:15,299 DEBUG [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038 2024-12-09T03:48:15,300 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [55d0183f16d2,35501,1733716094084] 2024-12-09T03:48:15,300 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [55d0183f16d2,39859,1733716093905] 2024-12-09T03:48:15,301 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [55d0183f16d2,45651,1733716094038] 2024-12-09T03:48:15,328 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T03:48:15,328 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T03:48:15,328 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T03:48:15,347 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T03:48:15,347 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T03:48:15,347 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T03:48:15,353 INFO [RS:0;55d0183f16d2:39859 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T03:48:15,353 INFO [RS:2;55d0183f16d2:35501 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T03:48:15,353 INFO [RS:1;55d0183f16d2:45651 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T03:48:15,354 INFO [RS:1;55d0183f16d2:45651 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,354 INFO [RS:0;55d0183f16d2:39859 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,354 INFO [RS:2;55d0183f16d2:35501 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,355 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T03:48:15,355 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T03:48:15,356 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T03:48:15,361 INFO [RS:1;55d0183f16d2:45651 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T03:48:15,361 INFO [RS:2;55d0183f16d2:35501 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T03:48:15,361 INFO [RS:0;55d0183f16d2:39859 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T03:48:15,362 INFO [RS:2;55d0183f16d2:35501 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,362 INFO [RS:1;55d0183f16d2:45651 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,362 INFO [RS:0;55d0183f16d2:39859 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,363 DEBUG [RS:2;55d0183f16d2:35501 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,363 DEBUG [RS:1;55d0183f16d2:45651 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,363 DEBUG [RS:0;55d0183f16d2:39859 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,363 DEBUG [RS:2;55d0183f16d2:35501 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,363 DEBUG [RS:1;55d0183f16d2:45651 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,363 DEBUG [RS:0;55d0183f16d2:39859 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,363 DEBUG [RS:2;55d0183f16d2:35501 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,363 DEBUG [RS:1;55d0183f16d2:45651 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,363 DEBUG [RS:0;55d0183f16d2:39859 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,363 DEBUG [RS:2;55d0183f16d2:35501 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,363 DEBUG [RS:1;55d0183f16d2:45651 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,363 DEBUG [RS:0;55d0183f16d2:39859 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,363 DEBUG [RS:2;55d0183f16d2:35501 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,363 DEBUG [RS:1;55d0183f16d2:45651 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,363 DEBUG [RS:0;55d0183f16d2:39859 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,363 DEBUG [RS:2;55d0183f16d2:35501 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/55d0183f16d2:0, corePoolSize=2, maxPoolSize=2 2024-12-09T03:48:15,363 DEBUG [RS:1;55d0183f16d2:45651 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/55d0183f16d2:0, corePoolSize=2, maxPoolSize=2 2024-12-09T03:48:15,363 DEBUG [RS:2;55d0183f16d2:35501 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,363 DEBUG [RS:0;55d0183f16d2:39859 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/55d0183f16d2:0, corePoolSize=2, maxPoolSize=2 2024-12-09T03:48:15,363 DEBUG [RS:1;55d0183f16d2:45651 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,364 DEBUG [RS:2;55d0183f16d2:35501 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,364 DEBUG [RS:0;55d0183f16d2:39859 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,364 DEBUG [RS:2;55d0183f16d2:35501 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,364 DEBUG [RS:1;55d0183f16d2:45651 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,364 DEBUG [RS:2;55d0183f16d2:35501 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,364 DEBUG [RS:0;55d0183f16d2:39859 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,364 DEBUG [RS:1;55d0183f16d2:45651 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,364 DEBUG [RS:2;55d0183f16d2:35501 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,364 DEBUG [RS:2;55d0183f16d2:35501 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,364 DEBUG [RS:0;55d0183f16d2:39859 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,364 DEBUG [RS:1;55d0183f16d2:45651 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,364 DEBUG [RS:2;55d0183f16d2:35501 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/55d0183f16d2:0, corePoolSize=3, maxPoolSize=3 2024-12-09T03:48:15,364 DEBUG [RS:0;55d0183f16d2:39859 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,364 DEBUG [RS:1;55d0183f16d2:45651 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,364 DEBUG [RS:2;55d0183f16d2:35501 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/55d0183f16d2:0, corePoolSize=3, maxPoolSize=3 2024-12-09T03:48:15,364 DEBUG [RS:0;55d0183f16d2:39859 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,364 DEBUG [RS:1;55d0183f16d2:45651 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,365 DEBUG [RS:0;55d0183f16d2:39859 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:48:15,365 DEBUG [RS:1;55d0183f16d2:45651 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/55d0183f16d2:0, corePoolSize=3, maxPoolSize=3 2024-12-09T03:48:15,365 DEBUG [RS:1;55d0183f16d2:45651 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/55d0183f16d2:0, corePoolSize=3, maxPoolSize=3 2024-12-09T03:48:15,365 DEBUG [RS:0;55d0183f16d2:39859 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/55d0183f16d2:0, corePoolSize=3, maxPoolSize=3 2024-12-09T03:48:15,365 DEBUG [RS:0;55d0183f16d2:39859 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/55d0183f16d2:0, corePoolSize=3, maxPoolSize=3 2024-12-09T03:48:15,369 INFO [RS:2;55d0183f16d2:35501 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,369 INFO [RS:2;55d0183f16d2:35501 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,369 INFO [RS:2;55d0183f16d2:35501 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,369 INFO [RS:1;55d0183f16d2:45651 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,369 INFO [RS:2;55d0183f16d2:35501 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,369 INFO [RS:0;55d0183f16d2:39859 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,369 INFO [RS:1;55d0183f16d2:45651 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,369 INFO [RS:2;55d0183f16d2:35501 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,369 INFO [RS:0;55d0183f16d2:39859 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,369 INFO [RS:1;55d0183f16d2:45651 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,369 INFO [RS:2;55d0183f16d2:35501 {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,35501,1733716094084-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T03:48:15,369 INFO [RS:0;55d0183f16d2:39859 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,369 INFO [RS:1;55d0183f16d2:45651 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,369 INFO [RS:0;55d0183f16d2:39859 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,369 INFO [RS:1;55d0183f16d2:45651 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,370 INFO [RS:0;55d0183f16d2:39859 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,370 INFO [RS:1;55d0183f16d2:45651 {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,45651,1733716094038-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T03:48:15,370 INFO [RS:0;55d0183f16d2:39859 {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,39859,1733716093905-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T03:48:15,390 WARN [55d0183f16d2:34431 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T03:48:15,399 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T03:48:15,399 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T03:48:15,399 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T03:48:15,402 INFO [RS:1;55d0183f16d2:45651 {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,45651,1733716094038-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,402 INFO [RS:0;55d0183f16d2:39859 {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,39859,1733716093905-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,402 INFO [RS:2;55d0183f16d2:35501 {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,35501,1733716094084-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,402 INFO [RS:1;55d0183f16d2:45651 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,402 INFO [RS:0;55d0183f16d2:39859 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,402 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.Replication(171): 55d0183f16d2,39859,1733716093905 started 2024-12-09T03:48:15,402 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.Replication(171): 55d0183f16d2,45651,1733716094038 started 2024-12-09T03:48:15,402 INFO [RS:2;55d0183f16d2:35501 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,403 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.Replication(171): 55d0183f16d2,35501,1733716094084 started 2024-12-09T03:48:15,433 INFO [RS:0;55d0183f16d2:39859 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,433 INFO [RS:2;55d0183f16d2:35501 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,433 INFO [RS:1;55d0183f16d2:45651 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:15,433 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(1482): Serving as 55d0183f16d2,45651,1733716094038, RpcServer on 55d0183f16d2/172.17.0.2:45651, sessionid=0x1019556106d0002 2024-12-09T03:48:15,433 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.HRegionServer(1482): Serving as 55d0183f16d2,39859,1733716093905, RpcServer on 55d0183f16d2/172.17.0.2:39859, sessionid=0x1019556106d0001 2024-12-09T03:48:15,433 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(1482): Serving as 55d0183f16d2,35501,1733716094084, RpcServer on 55d0183f16d2/172.17.0.2:35501, sessionid=0x1019556106d0003 2024-12-09T03:48:15,434 DEBUG [RS:0;55d0183f16d2:39859 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T03:48:15,434 DEBUG [RS:1;55d0183f16d2:45651 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T03:48:15,434 DEBUG [RS:2;55d0183f16d2:35501 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T03:48:15,435 DEBUG [RS:1;55d0183f16d2:45651 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 55d0183f16d2,45651,1733716094038 2024-12-09T03:48:15,435 DEBUG [RS:0;55d0183f16d2:39859 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 55d0183f16d2,39859,1733716093905 2024-12-09T03:48:15,435 DEBUG [RS:2;55d0183f16d2:35501 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 55d0183f16d2,35501,1733716094084 2024-12-09T03:48:15,435 DEBUG [RS:2;55d0183f16d2:35501 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '55d0183f16d2,35501,1733716094084' 2024-12-09T03:48:15,435 DEBUG [RS:1;55d0183f16d2:45651 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '55d0183f16d2,45651,1733716094038' 2024-12-09T03:48:15,435 DEBUG [RS:0;55d0183f16d2:39859 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '55d0183f16d2,39859,1733716093905' 2024-12-09T03:48:15,435 DEBUG [RS:1;55d0183f16d2:45651 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T03:48:15,435 DEBUG [RS:0;55d0183f16d2:39859 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T03:48:15,435 DEBUG [RS:2;55d0183f16d2:35501 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T03:48:15,436 DEBUG [RS:2;55d0183f16d2:35501 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T03:48:15,436 DEBUG [RS:0;55d0183f16d2:39859 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T03:48:15,436 DEBUG [RS:1;55d0183f16d2:45651 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T03:48:15,437 DEBUG [RS:1;55d0183f16d2:45651 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T03:48:15,437 DEBUG [RS:0;55d0183f16d2:39859 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T03:48:15,437 DEBUG [RS:2;55d0183f16d2:35501 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T03:48:15,437 DEBUG [RS:1;55d0183f16d2:45651 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T03:48:15,437 DEBUG [RS:2;55d0183f16d2:35501 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T03:48:15,437 DEBUG [RS:0;55d0183f16d2:39859 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T03:48:15,437 DEBUG [RS:1;55d0183f16d2:45651 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 55d0183f16d2,45651,1733716094038 2024-12-09T03:48:15,437 DEBUG [RS:2;55d0183f16d2:35501 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 55d0183f16d2,35501,1733716094084 2024-12-09T03:48:15,437 DEBUG [RS:0;55d0183f16d2:39859 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 55d0183f16d2,39859,1733716093905 2024-12-09T03:48:15,437 DEBUG [RS:2;55d0183f16d2:35501 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '55d0183f16d2,35501,1733716094084' 2024-12-09T03:48:15,437 DEBUG [RS:0;55d0183f16d2:39859 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '55d0183f16d2,39859,1733716093905' 2024-12-09T03:48:15,437 DEBUG [RS:1;55d0183f16d2:45651 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '55d0183f16d2,45651,1733716094038' 2024-12-09T03:48:15,437 DEBUG [RS:2;55d0183f16d2:35501 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T03:48:15,437 DEBUG [RS:0;55d0183f16d2:39859 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T03:48:15,437 DEBUG [RS:1;55d0183f16d2:45651 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T03:48:15,438 DEBUG [RS:2;55d0183f16d2:35501 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T03:48:15,438 DEBUG [RS:0;55d0183f16d2:39859 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T03:48:15,439 DEBUG [RS:2;55d0183f16d2:35501 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T03:48:15,439 DEBUG [RS:0;55d0183f16d2:39859 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T03:48:15,439 INFO [RS:2;55d0183f16d2:35501 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T03:48:15,439 INFO [RS:0;55d0183f16d2:39859 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T03:48:15,439 INFO [RS:2;55d0183f16d2:35501 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T03:48:15,439 INFO [RS:0;55d0183f16d2:39859 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T03:48:15,440 DEBUG [RS:1;55d0183f16d2:45651 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T03:48:15,441 DEBUG [RS:1;55d0183f16d2:45651 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T03:48:15,441 INFO [RS:1;55d0183f16d2:45651 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T03:48:15,441 INFO [RS:1;55d0183f16d2:45651 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T03:48:15,545 INFO [RS:0;55d0183f16d2:39859 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:48:15,545 INFO [RS:2;55d0183f16d2:35501 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:48:15,545 INFO [RS:1;55d0183f16d2:45651 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:48:15,548 INFO [RS:1;55d0183f16d2:45651 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=55d0183f16d2%2C45651%2C1733716094038, suffix=, logDir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038, archiveDir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/oldWALs, maxLogs=32 2024-12-09T03:48:15,548 INFO [RS:0;55d0183f16d2:39859 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=55d0183f16d2%2C39859%2C1733716093905, suffix=, logDir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,39859,1733716093905, archiveDir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/oldWALs, maxLogs=32 2024-12-09T03:48:15,548 INFO [RS:2;55d0183f16d2:35501 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=55d0183f16d2%2C35501%2C1733716094084, suffix=, logDir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,35501,1733716094084, archiveDir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/oldWALs, maxLogs=32 2024-12-09T03:48:15,568 DEBUG [RS:2;55d0183f16d2:35501 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,35501,1733716094084/55d0183f16d2%2C35501%2C1733716094084.1733716095554, exclude list is [], retry=0 2024-12-09T03:48:15,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:15,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:15,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:15,575 DEBUG [RS:1;55d0183f16d2:45651 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038/55d0183f16d2%2C45651%2C1733716094038.1733716095554, exclude list is [], retry=0 2024-12-09T03:48:15,575 DEBUG [RS:0;55d0183f16d2:39859 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,39859,1733716093905/55d0183f16d2%2C39859%2C1733716093905.1733716095554, exclude list is [], retry=0 2024-12-09T03:48:15,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:15,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:15,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:15,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:15,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:15,583 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:15,609 INFO [RS:2;55d0183f16d2:35501 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,35501,1733716094084/55d0183f16d2%2C35501%2C1733716094084.1733716095554 2024-12-09T03:48:15,610 DEBUG [RS:2;55d0183f16d2:35501 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:38745:38745)] 2024-12-09T03:48:15,619 INFO [RS:1;55d0183f16d2:45651 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038/55d0183f16d2%2C45651%2C1733716094038.1733716095554 2024-12-09T03:48:15,619 INFO [RS:0;55d0183f16d2:39859 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,39859,1733716093905/55d0183f16d2%2C39859%2C1733716093905.1733716095554 2024-12-09T03:48:15,619 DEBUG [RS:1;55d0183f16d2:45651 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:36521:36521)] 2024-12-09T03:48:15,620 DEBUG [RS:0;55d0183f16d2:39859 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:37281:37281)] 2024-12-09T03:48:15,643 DEBUG [55d0183f16d2:34431 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-09T03:48:15,651 DEBUG [55d0183f16d2:34431 {}] balancer.BalancerClusterState(204): Hosts are {55d0183f16d2=0} racks are {/default-rack=0} 2024-12-09T03:48:15,658 DEBUG [55d0183f16d2:34431 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T03:48:15,658 DEBUG [55d0183f16d2:34431 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T03:48:15,658 DEBUG [55d0183f16d2:34431 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-09T03:48:15,658 DEBUG [55d0183f16d2:34431 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T03:48:15,658 DEBUG [55d0183f16d2:34431 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T03:48:15,658 DEBUG [55d0183f16d2:34431 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T03:48:15,658 INFO [55d0183f16d2:34431 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T03:48:15,658 INFO [55d0183f16d2:34431 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T03:48:15,658 INFO [55d0183f16d2:34431 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T03:48:15,658 DEBUG [55d0183f16d2:34431 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T03:48:15,666 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=55d0183f16d2,35501,1733716094084 2024-12-09T03:48:15,671 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 55d0183f16d2,35501,1733716094084, state=OPENING 2024-12-09T03:48:15,677 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T03:48:15,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:15,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:15,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45651-0x1019556106d0002, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:15,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:15,679 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T03:48:15,679 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T03:48:15,679 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T03:48:15,679 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T03:48:15,681 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T03:48:15,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=55d0183f16d2,35501,1733716094084}] 2024-12-09T03:48:15,858 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T03:48:15,860 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53921, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T03:48:15,873 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T03:48:15,874 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:15,874 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T03:48:15,877 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=55d0183f16d2%2C35501%2C1733716094084.meta, suffix=.meta, logDir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,35501,1733716094084, archiveDir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/oldWALs, maxLogs=32 2024-12-09T03:48:15,893 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,35501,1733716094084/55d0183f16d2%2C35501%2C1733716094084.meta.1733716095879.meta, exclude list is [], retry=0 2024-12-09T03:48:15,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:15,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:15,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:15,932 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,35501,1733716094084/55d0183f16d2%2C35501%2C1733716094084.meta.1733716095879.meta 2024-12-09T03:48:15,933 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:37281:37281)] 2024-12-09T03:48:15,933 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:15,935 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T03:48:15,939 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T03:48:15,946 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T03:48:15,952 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T03:48:15,953 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:15,953 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T03:48:15,953 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T03:48:15,957 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T03:48:15,958 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T03:48:15,959 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:15,959 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:48:15,960 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T03:48:15,961 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T03:48:15,961 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:15,962 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:48:15,962 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T03:48:15,963 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T03:48:15,963 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:15,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:48:15,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T03:48:15,965 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T03:48:15,965 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:15,966 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:48:15,966 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T03:48:15,968 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740 2024-12-09T03:48:15,970 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740 2024-12-09T03:48:15,973 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T03:48:15,973 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T03:48:15,973 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T03:48:15,976 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T03:48:15,978 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62498348, jitterRate=-0.06870204210281372}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T03:48:15,978 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T03:48:15,980 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733716095954Writing region info on filesystem at 1733716095954Initializing all the Stores at 1733716095956 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733716095956Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733716095957 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716095957Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733716095957Cleaning up temporary data from old regions at 1733716095973 (+16 ms)Running coprocessor post-open hooks at 1733716095978 (+5 ms)Region opened successfully at 1733716095980 (+2 ms) 2024-12-09T03:48:15,990 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733716095849 2024-12-09T03:48:16,002 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T03:48:16,003 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T03:48:16,005 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=55d0183f16d2,35501,1733716094084 2024-12-09T03:48:16,007 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 55d0183f16d2,35501,1733716094084, state=OPEN 2024-12-09T03:48:16,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T03:48:16,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T03:48:16,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T03:48:16,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45651-0x1019556106d0002, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T03:48:16,012 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T03:48:16,012 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T03:48:16,012 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T03:48:16,012 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T03:48:16,013 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=55d0183f16d2,35501,1733716094084 2024-12-09T03:48:16,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T03:48:16,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=55d0183f16d2,35501,1733716094084 in 331 msec 2024-12-09T03:48:16,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T03:48:16,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 795 msec 2024-12-09T03:48:16,027 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T03:48:16,027 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T03:48:16,051 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T03:48:16,052 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=55d0183f16d2,35501,1733716094084, seqNum=-1] 2024-12-09T03:48:16,074 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T03:48:16,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49987, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T03:48:16,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1020 sec 2024-12-09T03:48:16,096 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733716096096, completionTime=-1 2024-12-09T03:48:16,098 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-09T03:48:16,098 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T03:48:16,127 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-09T03:48:16,127 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733716156127 2024-12-09T03:48:16,127 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733716216127 2024-12-09T03:48:16,128 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-12-09T03:48:16,129 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-09T03:48:16,137 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,34431,1733716093194-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:16,137 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,34431,1733716093194-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:16,137 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,34431,1733716093194-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:16,140 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-55d0183f16d2:34431, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:16,140 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:16,141 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:16,147 DEBUG [master/55d0183f16d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T03:48:16,176 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.013sec 2024-12-09T03:48:16,178 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T03:48:16,179 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T03:48:16,180 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T03:48:16,181 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T03:48:16,181 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T03:48:16,181 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,34431,1733716093194-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T03:48:16,182 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,34431,1733716093194-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T03:48:16,186 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T03:48:16,187 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T03:48:16,187 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,34431,1733716093194-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:16,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51245b74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:48:16,227 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 55d0183f16d2,34431,-1 for getting cluster id 2024-12-09T03:48:16,231 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T03:48:16,241 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '79d41f04-398a-445f-90b8-79288500355c' 2024-12-09T03:48:16,245 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T03:48:16,245 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "79d41f04-398a-445f-90b8-79288500355c" 2024-12-09T03:48:16,248 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f43cf6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:48:16,248 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [55d0183f16d2,34431,-1] 2024-12-09T03:48:16,251 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T03:48:16,253 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:48:16,254 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T03:48:16,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4010c1c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:48:16,257 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T03:48:16,265 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=55d0183f16d2,35501,1733716094084, seqNum=-1] 2024-12-09T03:48:16,266 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T03:48:16,268 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T03:48:16,295 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=55d0183f16d2,34431,1733716093194 2024-12-09T03:48:16,296 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:35437/hbase 2024-12-09T03:48:16,312 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=360, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=242, ProcessCount=11, AvailableMemoryMB=12191 2024-12-09T03:48:16,332 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:16,336 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:16,337 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:48:16,343 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-74140024, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/hregion-74140024, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:16,361 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-74140024/hregion-74140024.1733716096345, exclude list is [], retry=0 2024-12-09T03:48:16,365 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:16,365 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:16,366 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:16,405 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-74140024/hregion-74140024.1733716096345 2024-12-09T03:48:16,405 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:37281:37281)] 2024-12-09T03:48:16,406 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7b58e5ccf9838bc025b889d3a1b38784, NAME => 'testReplayEditsWrittenIntoWAL,,1733716096333.7b58e5ccf9838bc025b889d3a1b38784.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35437/hbase 2024-12-09T03:48:16,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741838_1014 (size=64) 2024-12-09T03:48:16,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741838_1014 (size=64) 2024-12-09T03:48:16,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741838_1014 (size=64) 2024-12-09T03:48:16,428 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733716096333.7b58e5ccf9838bc025b889d3a1b38784.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:16,430 INFO [StoreOpener-7b58e5ccf9838bc025b889d3a1b38784-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7b58e5ccf9838bc025b889d3a1b38784 2024-12-09T03:48:16,433 INFO [StoreOpener-7b58e5ccf9838bc025b889d3a1b38784-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b58e5ccf9838bc025b889d3a1b38784 columnFamilyName a 2024-12-09T03:48:16,433 DEBUG [StoreOpener-7b58e5ccf9838bc025b889d3a1b38784-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:16,434 INFO [StoreOpener-7b58e5ccf9838bc025b889d3a1b38784-1 {}] regionserver.HStore(327): Store=7b58e5ccf9838bc025b889d3a1b38784/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:16,434 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7b58e5ccf9838bc025b889d3a1b38784 2024-12-09T03:48:16,435 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784 2024-12-09T03:48:16,436 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784 2024-12-09T03:48:16,437 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7b58e5ccf9838bc025b889d3a1b38784 2024-12-09T03:48:16,437 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7b58e5ccf9838bc025b889d3a1b38784 2024-12-09T03:48:16,440 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7b58e5ccf9838bc025b889d3a1b38784 2024-12-09T03:48:16,444 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:48:16,444 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7b58e5ccf9838bc025b889d3a1b38784; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73655859, jitterRate=0.09755782783031464}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T03:48:16,446 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7b58e5ccf9838bc025b889d3a1b38784: Writing region info on filesystem at 1733716096428Initializing all the Stores at 1733716096430 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716096430Cleaning up temporary data from old regions at 1733716096437 (+7 ms)Region opened successfully at 1733716096446 (+9 ms) 2024-12-09T03:48:16,446 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7b58e5ccf9838bc025b889d3a1b38784, disabling compactions & flushes 2024-12-09T03:48:16,446 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733716096333.7b58e5ccf9838bc025b889d3a1b38784. 2024-12-09T03:48:16,446 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733716096333.7b58e5ccf9838bc025b889d3a1b38784. 2024-12-09T03:48:16,446 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733716096333.7b58e5ccf9838bc025b889d3a1b38784. after waiting 0 ms 2024-12-09T03:48:16,446 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733716096333.7b58e5ccf9838bc025b889d3a1b38784. 2024-12-09T03:48:16,447 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733716096333.7b58e5ccf9838bc025b889d3a1b38784. 2024-12-09T03:48:16,447 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7b58e5ccf9838bc025b889d3a1b38784: Waiting for close lock at 1733716096446Disabling compacts and flushes for region at 1733716096446Disabling writes for close at 1733716096446Writing region close event to WAL at 1733716096447 (+1 ms)Closed at 1733716096447 2024-12-09T03:48:16,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741837_1013 (size=175) 2024-12-09T03:48:16,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741837_1013 (size=175) 2024-12-09T03:48:16,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741837_1013 (size=175) 2024-12-09T03:48:16,463 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T03:48:16,463 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-74140024:(num 1733716096345) 2024-12-09T03:48:16,466 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T03:48:16,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741839_1015 (size=469) 2024-12-09T03:48:16,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741839_1015 (size=469) 2024-12-09T03:48:16,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741839_1015 (size=469) 2024-12-09T03:48:16,517 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T03:48:16,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741840_1016 (size=322) 2024-12-09T03:48:16,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741840_1016 (size=322) 2024-12-09T03:48:16,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741840_1016 (size=322) 2024-12-09T03:48:16,609 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-1, size=469 (469bytes) 2024-12-09T03:48:16,610 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-09T03:48:16,610 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-09T03:48:16,610 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-1 2024-12-09T03:48:16,617 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-1 after 5ms 2024-12-09T03:48:16,660 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:16,661 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-1 took 54ms 2024-12-09T03:48:16,667 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-1 so closing down 2024-12-09T03:48:16,667 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T03:48:16,671 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-09T03:48:16,708 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/0000000000000000001-wal-1.temp 2024-12-09T03:48:16,709 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T03:48:16,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741841_1017 (size=469) 2024-12-09T03:48:16,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741841_1017 (size=469) 2024-12-09T03:48:16,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741841_1017 (size=469) 2024-12-09T03:48:16,734 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T03:48:16,738 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/0000000000000000002 2024-12-09T03:48:16,743 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 76 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-09T03:48:16,744 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-1, journal: Splitting hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-1, size=469 (469bytes) at 1733716096609Finishing writing output for hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-1 so closing down at 1733716096667 (+58 ms)Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/0000000000000000001-wal-1.temp at 1733716096708 (+41 ms)3 split writer threads finished at 1733716096709 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733716096734 (+25 ms)Rename recovered edits hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/0000000000000000002 at 1733716096738 (+4 ms)Processed 2 edits across 1 Regions in 76 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733716096743 (+5 ms) 2024-12-09T03:48:16,765 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-2, size=322 (322bytes) 2024-12-09T03:48:16,765 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-2 2024-12-09T03:48:16,766 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-2 after 1ms 2024-12-09T03:48:16,807 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:16,807 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-2 took 43ms 2024-12-09T03:48:16,812 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-2 so closing down 2024-12-09T03:48:16,812 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T03:48:16,818 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-09T03:48:16,865 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/0000000000000000002-wal-2.temp 2024-12-09T03:48:16,865 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T03:48:16,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741842_1018 (size=322) 2024-12-09T03:48:16,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741842_1018 (size=322) 2024-12-09T03:48:16,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741842_1018 (size=322) 2024-12-09T03:48:16,882 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T03:48:16,917 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:16,920 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-12-09T03:48:16,922 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 114 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-09T03:48:16,923 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-2, journal: Splitting hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-2, size=322 (322bytes) at 1733716096765Finishing writing output for hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-2 so closing down at 1733716096812 (+47 ms)Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/0000000000000000002-wal-2.temp at 1733716096865 (+53 ms)3 split writer threads finished at 1733716096865Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733716096882 (+17 ms)Processed 1 edits across 1 Regions in 114 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733716096922 (+40 ms) 2024-12-09T03:48:16,923 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:16,926 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:16,948 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal.1733716096927, exclude list is [], retry=0 2024-12-09T03:48:16,953 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:16,954 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:16,954 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:16,986 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal.1733716096927 2024-12-09T03:48:16,986 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:36521:36521)] 2024-12-09T03:48:16,986 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b58e5ccf9838bc025b889d3a1b38784, NAME => 'testReplayEditsWrittenIntoWAL,,1733716096333.7b58e5ccf9838bc025b889d3a1b38784.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:16,987 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733716096333.7b58e5ccf9838bc025b889d3a1b38784.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:16,987 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7b58e5ccf9838bc025b889d3a1b38784 2024-12-09T03:48:16,987 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7b58e5ccf9838bc025b889d3a1b38784 2024-12-09T03:48:16,989 INFO [StoreOpener-7b58e5ccf9838bc025b889d3a1b38784-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7b58e5ccf9838bc025b889d3a1b38784 2024-12-09T03:48:16,991 INFO [StoreOpener-7b58e5ccf9838bc025b889d3a1b38784-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b58e5ccf9838bc025b889d3a1b38784 columnFamilyName a 2024-12-09T03:48:16,991 DEBUG [StoreOpener-7b58e5ccf9838bc025b889d3a1b38784-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:16,992 INFO [StoreOpener-7b58e5ccf9838bc025b889d3a1b38784-1 {}] regionserver.HStore(327): Store=7b58e5ccf9838bc025b889d3a1b38784/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:16,992 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7b58e5ccf9838bc025b889d3a1b38784 2024-12-09T03:48:16,993 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784 2024-12-09T03:48:16,996 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784 2024-12-09T03:48:16,997 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/0000000000000000002 2024-12-09T03:48:17,029 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:17,036 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/0000000000000000002 2024-12-09T03:48:17,040 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7b58e5ccf9838bc025b889d3a1b38784 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-09T03:48:17,133 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/.tmp/a/c4967eaf15d8412ea091531b92545270 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733716096464/Put/seqid=0 2024-12-09T03:48:17,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741844_1020 (size=5170) 2024-12-09T03:48:17,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741844_1020 (size=5170) 2024-12-09T03:48:17,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741844_1020 (size=5170) 2024-12-09T03:48:17,148 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/.tmp/a/c4967eaf15d8412ea091531b92545270 2024-12-09T03:48:17,197 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/.tmp/a/c4967eaf15d8412ea091531b92545270 as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/a/c4967eaf15d8412ea091531b92545270 2024-12-09T03:48:17,206 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/a/c4967eaf15d8412ea091531b92545270, entries=2, sequenceid=2, filesize=5.0 K 2024-12-09T03:48:17,213 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 7b58e5ccf9838bc025b889d3a1b38784 in 171ms, sequenceid=2, compaction requested=false; wal=null 2024-12-09T03:48:17,215 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/0000000000000000002 2024-12-09T03:48:17,216 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7b58e5ccf9838bc025b889d3a1b38784 2024-12-09T03:48:17,216 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7b58e5ccf9838bc025b889d3a1b38784 2024-12-09T03:48:17,219 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7b58e5ccf9838bc025b889d3a1b38784 2024-12-09T03:48:17,222 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/7b58e5ccf9838bc025b889d3a1b38784/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-09T03:48:17,223 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7b58e5ccf9838bc025b889d3a1b38784; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63322841, jitterRate=-0.056416139006614685}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T03:48:17,224 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7b58e5ccf9838bc025b889d3a1b38784: Writing region info on filesystem at 1733716096987Initializing all the Stores at 1733716096989 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716096989Obtaining lock to block concurrent updates at 1733716097040 (+51 ms)Preparing flush snapshotting stores in 7b58e5ccf9838bc025b889d3a1b38784 at 1733716097040Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733716096333.7b58e5ccf9838bc025b889d3a1b38784., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733716097043 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733716096333.7b58e5ccf9838bc025b889d3a1b38784. at 1733716097043Flushing 7b58e5ccf9838bc025b889d3a1b38784/a: creating writer at 1733716097044 (+1 ms)Flushing 7b58e5ccf9838bc025b889d3a1b38784/a: appending metadata at 1733716097123 (+79 ms)Flushing 7b58e5ccf9838bc025b889d3a1b38784/a: closing flushed file at 1733716097126 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a42d9b4: reopening flushed file at 1733716097196 (+70 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 7b58e5ccf9838bc025b889d3a1b38784 in 171ms, sequenceid=2, compaction requested=false; wal=null at 1733716097213 (+17 ms)Cleaning up temporary data from old regions at 1733716097216 (+3 ms)Region opened successfully at 1733716097224 (+8 ms) 2024-12-09T03:48:17,251 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=372 (was 360) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:35586 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:46580 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:35618 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2031904063-172.17.0.2-1733716090120:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:40476 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2031904063-172.17.0.2-1733716090120:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:46448 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:40286 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2031904063-172.17.0.2-1733716090120:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:35437/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=242 (was 242), ProcessCount=11 (was 11), AvailableMemoryMB=12051 (was 12191) 2024-12-09T03:48:17,262 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=372, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=242, ProcessCount=11, AvailableMemoryMB=12050 2024-12-09T03:48:17,278 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:17,280 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:17,281 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:48:17,285 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-02955107, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/hregion-02955107, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:17,298 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-02955107/hregion-02955107.1733716097285, exclude list is [], retry=0 2024-12-09T03:48:17,302 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:17,302 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:17,302 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:17,341 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-02955107/hregion-02955107.1733716097285 2024-12-09T03:48:17,341 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:38745:38745)] 2024-12-09T03:48:17,341 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d98a600bdcad7131020a506026d7b049, NAME => 'testReplayEditsWrittenIntoWAL,,1733716097279.d98a600bdcad7131020a506026d7b049.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35437/hbase 2024-12-09T03:48:17,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741846_1022 (size=64) 2024-12-09T03:48:17,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741846_1022 (size=64) 2024-12-09T03:48:17,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741846_1022 (size=64) 2024-12-09T03:48:17,355 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733716097279.d98a600bdcad7131020a506026d7b049.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:17,357 INFO [StoreOpener-d98a600bdcad7131020a506026d7b049-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d98a600bdcad7131020a506026d7b049 2024-12-09T03:48:17,359 INFO [StoreOpener-d98a600bdcad7131020a506026d7b049-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d98a600bdcad7131020a506026d7b049 columnFamilyName a 2024-12-09T03:48:17,359 DEBUG [StoreOpener-d98a600bdcad7131020a506026d7b049-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:17,360 INFO [StoreOpener-d98a600bdcad7131020a506026d7b049-1 {}] regionserver.HStore(327): Store=d98a600bdcad7131020a506026d7b049/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:17,360 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d98a600bdcad7131020a506026d7b049 2024-12-09T03:48:17,361 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049 2024-12-09T03:48:17,362 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049 2024-12-09T03:48:17,362 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d98a600bdcad7131020a506026d7b049 2024-12-09T03:48:17,362 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d98a600bdcad7131020a506026d7b049 2024-12-09T03:48:17,365 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d98a600bdcad7131020a506026d7b049 2024-12-09T03:48:17,368 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:48:17,368 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d98a600bdcad7131020a506026d7b049; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68864687, jitterRate=0.026163801550865173}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T03:48:17,369 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d98a600bdcad7131020a506026d7b049: Writing region info on filesystem at 1733716097355Initializing all the Stores at 1733716097356 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716097356Cleaning up temporary data from old regions at 1733716097362 (+6 ms)Region opened successfully at 1733716097368 (+6 ms) 2024-12-09T03:48:17,369 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d98a600bdcad7131020a506026d7b049, disabling compactions & flushes 2024-12-09T03:48:17,369 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733716097279.d98a600bdcad7131020a506026d7b049. 2024-12-09T03:48:17,369 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733716097279.d98a600bdcad7131020a506026d7b049. 2024-12-09T03:48:17,369 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733716097279.d98a600bdcad7131020a506026d7b049. after waiting 0 ms 2024-12-09T03:48:17,369 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733716097279.d98a600bdcad7131020a506026d7b049. 2024-12-09T03:48:17,369 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733716097279.d98a600bdcad7131020a506026d7b049. 2024-12-09T03:48:17,369 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d98a600bdcad7131020a506026d7b049: Waiting for close lock at 1733716097369Disabling compacts and flushes for region at 1733716097369Disabling writes for close at 1733716097369Writing region close event to WAL at 1733716097369Closed at 1733716097369 2024-12-09T03:48:17,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741845_1021 (size=175) 2024-12-09T03:48:17,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741845_1021 (size=175) 2024-12-09T03:48:17,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741845_1021 (size=175) 2024-12-09T03:48:17,376 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T03:48:17,376 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-02955107:(num 1733716097285) 2024-12-09T03:48:17,377 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T03:48:17,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741847_1023 (size=469) 2024-12-09T03:48:17,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741847_1023 (size=469) 2024-12-09T03:48:17,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741847_1023 (size=469) 2024-12-09T03:48:17,419 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T03:48:17,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741848_1024 (size=322) 2024-12-09T03:48:17,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741848_1024 (size=322) 2024-12-09T03:48:17,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741848_1024 (size=322) 2024-12-09T03:48:17,477 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-2, size=322 (322bytes) 2024-12-09T03:48:17,477 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-2 2024-12-09T03:48:17,478 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-2 after 0ms 2024-12-09T03:48:17,506 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:17,506 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-2 took 30ms 2024-12-09T03:48:17,508 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-2 so closing down 2024-12-09T03:48:17,508 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T03:48:17,510 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-09T03:48:17,547 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000002-wal-2.temp 2024-12-09T03:48:17,548 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T03:48:17,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741849_1025 (size=322) 2024-12-09T03:48:17,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741849_1025 (size=322) 2024-12-09T03:48:17,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741849_1025 (size=322) 2024-12-09T03:48:17,558 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T03:48:17,560 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000002 2024-12-09T03:48:17,560 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 54 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-09T03:48:17,561 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-2, journal: Splitting hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-2, size=322 (322bytes) at 1733716097477Finishing writing output for hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-2 so closing down at 1733716097508 (+31 ms)Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000002-wal-2.temp at 1733716097547 (+39 ms)3 split writer threads finished at 1733716097548 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733716097558 (+10 ms)Rename recovered edits hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000002 at 1733716097560 (+2 ms)Processed 1 edits across 1 Regions in 54 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733716097560 2024-12-09T03:48:17,575 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-1, size=469 (469bytes) 2024-12-09T03:48:17,575 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-1 2024-12-09T03:48:17,576 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-1 after 1ms 2024-12-09T03:48:17,605 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:17,605 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-1 took 30ms 2024-12-09T03:48:17,608 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-1 so closing down 2024-12-09T03:48:17,608 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T03:48:17,610 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-09T03:48:17,638 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000001-wal-1.temp 2024-12-09T03:48:17,638 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T03:48:17,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741850_1026 (size=469) 2024-12-09T03:48:17,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741850_1026 (size=469) 2024-12-09T03:48:17,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741850_1026 (size=469) 2024-12-09T03:48:17,649 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T03:48:17,689 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:17,692 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000002, length=322 2024-12-09T03:48:17,694 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000002 2024-12-09T03:48:17,695 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 89 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-09T03:48:17,695 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-1, journal: Splitting hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-1, size=469 (469bytes) at 1733716097575Finishing writing output for hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-1 so closing down at 1733716097608 (+33 ms)Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000001-wal-1.temp at 1733716097638 (+30 ms)3 split writer threads finished at 1733716097638Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733716097649 (+11 ms)Rename recovered edits hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000002 at 1733716097695 (+46 ms)Processed 2 edits across 1 Regions in 89 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733716097695 2024-12-09T03:48:17,695 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:17,697 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:17,711 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal.1733716097698, exclude list is [], retry=0 2024-12-09T03:48:17,714 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:17,715 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:17,715 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:17,744 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal.1733716097698 2024-12-09T03:48:17,744 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:38745:38745)] 2024-12-09T03:48:17,744 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d98a600bdcad7131020a506026d7b049, NAME => 'testReplayEditsWrittenIntoWAL,,1733716097279.d98a600bdcad7131020a506026d7b049.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:17,744 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733716097279.d98a600bdcad7131020a506026d7b049.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:17,745 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d98a600bdcad7131020a506026d7b049 2024-12-09T03:48:17,745 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d98a600bdcad7131020a506026d7b049 2024-12-09T03:48:17,747 INFO [StoreOpener-d98a600bdcad7131020a506026d7b049-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d98a600bdcad7131020a506026d7b049 2024-12-09T03:48:17,749 INFO [StoreOpener-d98a600bdcad7131020a506026d7b049-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d98a600bdcad7131020a506026d7b049 columnFamilyName a 2024-12-09T03:48:17,749 DEBUG [StoreOpener-d98a600bdcad7131020a506026d7b049-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:17,750 INFO [StoreOpener-d98a600bdcad7131020a506026d7b049-1 {}] regionserver.HStore(327): Store=d98a600bdcad7131020a506026d7b049/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:17,750 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d98a600bdcad7131020a506026d7b049 2024-12-09T03:48:17,751 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049 2024-12-09T03:48:17,753 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049 2024-12-09T03:48:17,754 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000002 2024-12-09T03:48:17,782 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:17,784 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000002 2024-12-09T03:48:17,784 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d98a600bdcad7131020a506026d7b049 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-09T03:48:17,800 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/.tmp/a/d1509c0db73a4e9f81b6aec211ac5a08 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733716097376/Put/seqid=0 2024-12-09T03:48:17,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741852_1028 (size=5170) 2024-12-09T03:48:17,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741852_1028 (size=5170) 2024-12-09T03:48:17,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741852_1028 (size=5170) 2024-12-09T03:48:17,811 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/.tmp/a/d1509c0db73a4e9f81b6aec211ac5a08 2024-12-09T03:48:17,819 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/.tmp/a/d1509c0db73a4e9f81b6aec211ac5a08 as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/a/d1509c0db73a4e9f81b6aec211ac5a08 2024-12-09T03:48:17,828 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/a/d1509c0db73a4e9f81b6aec211ac5a08, entries=2, sequenceid=2, filesize=5.0 K 2024-12-09T03:48:17,828 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for d98a600bdcad7131020a506026d7b049 in 44ms, sequenceid=2, compaction requested=false; wal=null 2024-12-09T03:48:17,829 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/0000000000000000002 2024-12-09T03:48:17,830 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d98a600bdcad7131020a506026d7b049 2024-12-09T03:48:17,830 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d98a600bdcad7131020a506026d7b049 2024-12-09T03:48:17,833 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d98a600bdcad7131020a506026d7b049 2024-12-09T03:48:17,835 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/d98a600bdcad7131020a506026d7b049/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-09T03:48:17,836 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d98a600bdcad7131020a506026d7b049; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69930110, jitterRate=0.042039841413497925}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T03:48:17,837 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d98a600bdcad7131020a506026d7b049: Writing region info on filesystem at 1733716097745Initializing all the Stores at 1733716097746 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716097746Obtaining lock to block concurrent updates at 1733716097784 (+38 ms)Preparing flush snapshotting stores in d98a600bdcad7131020a506026d7b049 at 1733716097784Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733716097279.d98a600bdcad7131020a506026d7b049., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733716097784Flushing stores of testReplayEditsWrittenIntoWAL,,1733716097279.d98a600bdcad7131020a506026d7b049. at 1733716097784Flushing d98a600bdcad7131020a506026d7b049/a: creating writer at 1733716097784Flushing d98a600bdcad7131020a506026d7b049/a: appending metadata at 1733716097799 (+15 ms)Flushing d98a600bdcad7131020a506026d7b049/a: closing flushed file at 1733716097799Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@663e8269: reopening flushed file at 1733716097818 (+19 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for d98a600bdcad7131020a506026d7b049 in 44ms, sequenceid=2, compaction requested=false; wal=null at 1733716097828 (+10 ms)Cleaning up temporary data from old regions at 1733716097830 (+2 ms)Region opened successfully at 1733716097837 (+7 ms) 2024-12-09T03:48:17,860 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=382 (was 372) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:35586 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2031904063-172.17.0.2-1733716090120:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:46650 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:46448 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:40286 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:40528 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2031904063-172.17.0.2-1733716090120:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:35692 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2031904063-172.17.0.2-1733716090120:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=242 (was 242), ProcessCount=11 (was 11), AvailableMemoryMB=12036 (was 12050) 2024-12-09T03:48:17,872 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=382, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=242, ProcessCount=11, AvailableMemoryMB=12035 2024-12-09T03:48:17,888 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:17,891 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:17,891 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:48:17,895 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-81094935, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/hregion-81094935, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:17,910 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-81094935/hregion-81094935.1733716097896, exclude list is [], retry=0 2024-12-09T03:48:17,914 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:17,915 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:17,915 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:17,943 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-81094935/hregion-81094935.1733716097896 2024-12-09T03:48:17,944 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:36521:36521)] 2024-12-09T03:48:17,944 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1cf23ce7093cfac364e49afa143e25ce, NAME => 'testReplayEditsWrittenIntoWAL,,1733716097889.1cf23ce7093cfac364e49afa143e25ce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35437/hbase 2024-12-09T03:48:17,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741854_1030 (size=64) 2024-12-09T03:48:17,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741854_1030 (size=64) 2024-12-09T03:48:17,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741854_1030 (size=64) 2024-12-09T03:48:17,957 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733716097889.1cf23ce7093cfac364e49afa143e25ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:17,959 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:17,961 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1cf23ce7093cfac364e49afa143e25ce columnFamilyName a 2024-12-09T03:48:17,961 DEBUG [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:17,962 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] regionserver.HStore(327): Store=1cf23ce7093cfac364e49afa143e25ce/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:17,962 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:17,964 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1cf23ce7093cfac364e49afa143e25ce columnFamilyName b 2024-12-09T03:48:17,964 DEBUG [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:17,965 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] regionserver.HStore(327): Store=1cf23ce7093cfac364e49afa143e25ce/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:17,965 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:17,967 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1cf23ce7093cfac364e49afa143e25ce columnFamilyName c 2024-12-09T03:48:17,967 DEBUG [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:17,968 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] regionserver.HStore(327): Store=1cf23ce7093cfac364e49afa143e25ce/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:17,968 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:17,969 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:17,969 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:17,971 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:17,971 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:17,972 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:17,974 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:17,976 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:48:17,977 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1cf23ce7093cfac364e49afa143e25ce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59959738, jitterRate=-0.10653027892112732}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:17,977 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1cf23ce7093cfac364e49afa143e25ce: Writing region info on filesystem at 1733716097958Initializing all the Stores at 1733716097959 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716097959Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716097959Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716097959Cleaning up temporary data from old regions at 1733716097971 (+12 ms)Region opened successfully at 1733716097977 (+6 ms) 2024-12-09T03:48:17,978 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1cf23ce7093cfac364e49afa143e25ce, disabling compactions & flushes 2024-12-09T03:48:17,978 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733716097889.1cf23ce7093cfac364e49afa143e25ce. 2024-12-09T03:48:17,978 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733716097889.1cf23ce7093cfac364e49afa143e25ce. 2024-12-09T03:48:17,978 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733716097889.1cf23ce7093cfac364e49afa143e25ce. after waiting 0 ms 2024-12-09T03:48:17,978 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733716097889.1cf23ce7093cfac364e49afa143e25ce. 2024-12-09T03:48:17,978 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733716097889.1cf23ce7093cfac364e49afa143e25ce. 2024-12-09T03:48:17,978 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1cf23ce7093cfac364e49afa143e25ce: Waiting for close lock at 1733716097978Disabling compacts and flushes for region at 1733716097978Disabling writes for close at 1733716097978Writing region close event to WAL at 1733716097978Closed at 1733716097978 2024-12-09T03:48:17,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741853_1029 (size=175) 2024-12-09T03:48:17,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741853_1029 (size=175) 2024-12-09T03:48:17,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741853_1029 (size=175) 2024-12-09T03:48:17,985 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T03:48:17,985 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-81094935:(num 1733716097896) 2024-12-09T03:48:17,985 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:17,988 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:18,005 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988, exclude list is [], retry=0 2024-12-09T03:48:18,009 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:18,010 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:18,010 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:18,042 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988 2024-12-09T03:48:18,042 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:36521:36521)] 2024-12-09T03:48:18,282 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988, size=0 (0bytes) 2024-12-09T03:48:18,282 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988 might be still open, length is 0 2024-12-09T03:48:18,282 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988 2024-12-09T03:48:18,283 WARN [IPC Server handler 3 on default port 35437 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-09T03:48:18,284 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988 after 2ms 2024-12-09T03:48:21,138 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:35726 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39115:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35726 dst: /127.0.0.1:39115 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39115 remote=/127.0.0.1:35726]. Total timeout mills is 60000, 57099 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:21,139 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:46710 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36217:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46710 dst: /127.0.0.1:36217 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:21,139 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:40552 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45815:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40552 dst: /127.0.0.1:45815 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:21,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741855_1032 (size=497775) 2024-12-09T03:48:21,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741855_1032 (size=497775) 2024-12-09T03:48:21,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741855_1032 (size=497775) 2024-12-09T03:48:21,456 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T03:48:21,503 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T03:48:22,285 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988 after 4003ms 2024-12-09T03:48:22,316 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:22,318 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988 took 4036ms 2024-12-09T03:48:22,323 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733716097988.temp 2024-12-09T03:48:22,356 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/recovered.edits/0000000000000000001-wal.1733716097988.temp 2024-12-09T03:48:22,403 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988; continuing. 2024-12-09T03:48:22,403 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988 so closing down 2024-12-09T03:48:22,403 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T03:48:22,403 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T03:48:22,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741856_1033 (size=497783) 2024-12-09T03:48:22,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741856_1033 (size=497783) 2024-12-09T03:48:22,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741856_1033 (size=497783) 2024-12-09T03:48:22,408 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/recovered.edits/0000000000000000001-wal.1733716097988.temp (wrote 3002 edits, skipped 0 edits in 44 ms) 2024-12-09T03:48:22,410 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/recovered.edits/0000000000000000001-wal.1733716097988.temp to hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/recovered.edits/0000000000000003002 2024-12-09T03:48:22,411 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 92 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T03:48:22,411 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988, journal: Splitting hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988, size=0 (0bytes) at 1733716098282Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/recovered.edits/0000000000000000001-wal.1733716097988.temp at 1733716102356 (+4074 ms)Split 1024 edits, skipped 0 edits. at 1733716102367 (+11 ms)Split 2048 edits, skipped 0 edits. at 1733716102389 (+22 ms)Finishing writing output for hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988 so closing down at 1733716102403 (+14 ms)3 split writer threads finished at 1733716102403Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/recovered.edits/0000000000000000001-wal.1733716097988.temp (wrote 3002 edits, skipped 0 edits in 44 ms) at 1733716102408 (+5 ms)Rename recovered edits hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/recovered.edits/0000000000000000001-wal.1733716097988.temp to hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/recovered.edits/0000000000000003002 at 1733716102410 (+2 ms)Processed 3002 edits across 1 Regions in 92 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988, size=0, length=0, corrupted=false, cancelled=false at 1733716102411 (+1 ms) 2024-12-09T03:48:22,413 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988 to hdfs://localhost:35437/hbase/oldWALs/wal.1733716097988 2024-12-09T03:48:22,415 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/recovered.edits/0000000000000003002 2024-12-09T03:48:22,415 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:22,418 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:22,436 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716102419, exclude list is [], retry=0 2024-12-09T03:48:22,439 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:22,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:22,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:22,468 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716102419 2024-12-09T03:48:22,468 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:37281:37281)] 2024-12-09T03:48:22,468 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733716097889.1cf23ce7093cfac364e49afa143e25ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:22,470 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:22,471 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1cf23ce7093cfac364e49afa143e25ce columnFamilyName a 2024-12-09T03:48:22,471 DEBUG [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:22,472 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] regionserver.HStore(327): Store=1cf23ce7093cfac364e49afa143e25ce/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:22,472 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:22,473 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1cf23ce7093cfac364e49afa143e25ce columnFamilyName b 2024-12-09T03:48:22,473 DEBUG [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:22,474 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] regionserver.HStore(327): Store=1cf23ce7093cfac364e49afa143e25ce/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:22,474 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:22,475 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1cf23ce7093cfac364e49afa143e25ce columnFamilyName c 2024-12-09T03:48:22,475 DEBUG [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:22,475 INFO [StoreOpener-1cf23ce7093cfac364e49afa143e25ce-1 {}] regionserver.HStore(327): Store=1cf23ce7093cfac364e49afa143e25ce/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:22,476 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:22,476 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:22,478 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:22,479 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/recovered.edits/0000000000000003002 2024-12-09T03:48:22,508 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:22,549 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T03:48:22,934 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1cf23ce7093cfac364e49afa143e25ce 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T03:48:22,974 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/a/4017f9bf32484680a0cb5b3cc0e917b9 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733716098053/Put/seqid=0 2024-12-09T03:48:22,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741858_1035 (size=50463) 2024-12-09T03:48:22,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741858_1035 (size=50463) 2024-12-09T03:48:22,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741858_1035 (size=50463) 2024-12-09T03:48:22,984 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/a/4017f9bf32484680a0cb5b3cc0e917b9 2024-12-09T03:48:22,992 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/a/4017f9bf32484680a0cb5b3cc0e917b9 as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/a/4017f9bf32484680a0cb5b3cc0e917b9 2024-12-09T03:48:22,998 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/a/4017f9bf32484680a0cb5b3cc0e917b9, entries=754, sequenceid=754, filesize=49.3 K 2024-12-09T03:48:22,998 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 1cf23ce7093cfac364e49afa143e25ce in 64ms, sequenceid=754, compaction requested=false; wal=null 2024-12-09T03:48:23,009 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T03:48:23,009 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1cf23ce7093cfac364e49afa143e25ce 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T03:48:23,017 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/a/295cd8e5ceeb41b39c61d0cc691c32d2 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733716098104/Put/seqid=0 2024-12-09T03:48:23,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741859_1036 (size=20072) 2024-12-09T03:48:23,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741859_1036 (size=20072) 2024-12-09T03:48:23,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741859_1036 (size=20072) 2024-12-09T03:48:23,028 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/a/295cd8e5ceeb41b39c61d0cc691c32d2 2024-12-09T03:48:23,054 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/b/9ac4a3d7c7994be8bf127accbb8b6db0 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733716098127/Put/seqid=0 2024-12-09T03:48:23,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741860_1037 (size=35835) 2024-12-09T03:48:23,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741860_1037 (size=35835) 2024-12-09T03:48:23,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741860_1037 (size=35835) 2024-12-09T03:48:23,063 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/b/9ac4a3d7c7994be8bf127accbb8b6db0 2024-12-09T03:48:23,070 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/a/295cd8e5ceeb41b39c61d0cc691c32d2 as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/a/295cd8e5ceeb41b39c61d0cc691c32d2 2024-12-09T03:48:23,076 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/a/295cd8e5ceeb41b39c61d0cc691c32d2, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-09T03:48:23,077 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/b/9ac4a3d7c7994be8bf127accbb8b6db0 as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/b/9ac4a3d7c7994be8bf127accbb8b6db0 2024-12-09T03:48:23,083 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/b/9ac4a3d7c7994be8bf127accbb8b6db0, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-09T03:48:23,083 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 1cf23ce7093cfac364e49afa143e25ce in 74ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-09T03:48:23,093 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T03:48:23,093 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1cf23ce7093cfac364e49afa143e25ce 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T03:48:23,099 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/b/ac14f8e34d0d450694d914690a5fd485 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733716098153/Put/seqid=0 2024-12-09T03:48:23,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741861_1038 (size=35082) 2024-12-09T03:48:23,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741861_1038 (size=35082) 2024-12-09T03:48:23,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741861_1038 (size=35082) 2024-12-09T03:48:23,109 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/b/ac14f8e34d0d450694d914690a5fd485 2024-12-09T03:48:23,141 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/c/6cce431c48ca411b91cb836e1a43d5c6 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733716098186/Put/seqid=0 2024-12-09T03:48:23,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741862_1039 (size=20825) 2024-12-09T03:48:23,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741862_1039 (size=20825) 2024-12-09T03:48:23,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741862_1039 (size=20825) 2024-12-09T03:48:23,151 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/c/6cce431c48ca411b91cb836e1a43d5c6 2024-12-09T03:48:23,157 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/b/ac14f8e34d0d450694d914690a5fd485 as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/b/ac14f8e34d0d450694d914690a5fd485 2024-12-09T03:48:23,164 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/b/ac14f8e34d0d450694d914690a5fd485, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-09T03:48:23,165 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/c/6cce431c48ca411b91cb836e1a43d5c6 as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/c/6cce431c48ca411b91cb836e1a43d5c6 2024-12-09T03:48:23,170 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/c/6cce431c48ca411b91cb836e1a43d5c6, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-09T03:48:23,170 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 1cf23ce7093cfac364e49afa143e25ce in 77ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-09T03:48:23,178 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733716098235/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733716097889.1cf23ce7093cfac364e49afa143e25ce. 2024-12-09T03:48:23,180 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/recovered.edits/0000000000000003002 2024-12-09T03:48:23,181 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T03:48:23,181 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1cf23ce7093cfac364e49afa143e25ce 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-09T03:48:23,188 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/c/07f004efbf1d462e8705d3cbb7ee7989 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733716098194/Put/seqid=0 2024-12-09T03:48:23,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741863_1040 (size=50301) 2024-12-09T03:48:23,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741863_1040 (size=50301) 2024-12-09T03:48:23,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741863_1040 (size=50301) 2024-12-09T03:48:23,196 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/c/07f004efbf1d462e8705d3cbb7ee7989 2024-12-09T03:48:23,201 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 07f004efbf1d462e8705d3cbb7ee7989 2024-12-09T03:48:23,203 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/.tmp/c/07f004efbf1d462e8705d3cbb7ee7989 as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/c/07f004efbf1d462e8705d3cbb7ee7989 2024-12-09T03:48:23,208 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 07f004efbf1d462e8705d3cbb7ee7989 2024-12-09T03:48:23,209 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/c/07f004efbf1d462e8705d3cbb7ee7989, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-09T03:48:23,209 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 1cf23ce7093cfac364e49afa143e25ce in 28ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-09T03:48:23,210 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/recovered.edits/0000000000000003002 2024-12-09T03:48:23,212 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:23,212 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:23,213 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T03:48:23,216 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1cf23ce7093cfac364e49afa143e25ce 2024-12-09T03:48:23,222 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenIntoWAL/1cf23ce7093cfac364e49afa143e25ce/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-09T03:48:23,224 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1cf23ce7093cfac364e49afa143e25ce; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63600393, jitterRate=-0.05228029191493988}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T03:48:23,224 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1cf23ce7093cfac364e49afa143e25ce: Writing region info on filesystem at 1733716102469Initializing all the Stores at 1733716102470 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716102470Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716102470Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716102470Cleaning up temporary data from old regions at 1733716103212 (+742 ms)Region opened successfully at 1733716103224 (+12 ms) 2024-12-09T03:48:23,288 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1cf23ce7093cfac364e49afa143e25ce, disabling compactions & flushes 2024-12-09T03:48:23,288 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733716097889.1cf23ce7093cfac364e49afa143e25ce. 2024-12-09T03:48:23,288 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733716097889.1cf23ce7093cfac364e49afa143e25ce. 2024-12-09T03:48:23,288 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733716097889.1cf23ce7093cfac364e49afa143e25ce. after waiting 0 ms 2024-12-09T03:48:23,288 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733716097889.1cf23ce7093cfac364e49afa143e25ce. 2024-12-09T03:48:23,292 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733716097889.1cf23ce7093cfac364e49afa143e25ce. 2024-12-09T03:48:23,293 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1cf23ce7093cfac364e49afa143e25ce: Waiting for close lock at 1733716103287Disabling compacts and flushes for region at 1733716103287Disabling writes for close at 1733716103288 (+1 ms)Writing region close event to WAL at 1733716103292 (+4 ms)Closed at 1733716103292 2024-12-09T03:48:23,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741857_1034 (size=175) 2024-12-09T03:48:23,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741857_1034 (size=175) 2024-12-09T03:48:23,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741857_1034 (size=175) 2024-12-09T03:48:23,302 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T03:48:23,303 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733716102419) 2024-12-09T03:48:23,317 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=395 (was 382) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1419382495_22 at /127.0.0.1:39988 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:45919 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35437 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44623 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:44623 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1419382495_22 at /127.0.0.1:48380 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:35437 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1419382495_22 at /127.0.0.1:51420 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1419382495_22 at /127.0.0.1:39970 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45919 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=222 (was 242), ProcessCount=11 (was 11), AvailableMemoryMB=11970 (was 12035) 2024-12-09T03:48:23,329 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=395, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=222, ProcessCount=11, AvailableMemoryMB=11969 2024-12-09T03:48:23,346 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:23,348 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:23,349 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:48:23,352 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-93977570, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/hregion-93977570, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:23,366 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-93977570/hregion-93977570.1733716103353, exclude list is [], retry=0 2024-12-09T03:48:23,370 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:23,370 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:23,370 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:23,398 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-93977570/hregion-93977570.1733716103353 2024-12-09T03:48:23,399 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:38745:38745)] 2024-12-09T03:48:23,399 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => aad3b56713e2cfed2ad3d2a0e2facd58, NAME => 'test2727,,1733716103347.aad3b56713e2cfed2ad3d2a0e2facd58.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35437/hbase 2024-12-09T03:48:23,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741865_1042 (size=43) 2024-12-09T03:48:23,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741865_1042 (size=43) 2024-12-09T03:48:23,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741865_1042 (size=43) 2024-12-09T03:48:23,412 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733716103347.aad3b56713e2cfed2ad3d2a0e2facd58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:23,413 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:23,415 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aad3b56713e2cfed2ad3d2a0e2facd58 columnFamilyName a 2024-12-09T03:48:23,415 DEBUG [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:23,416 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] regionserver.HStore(327): Store=aad3b56713e2cfed2ad3d2a0e2facd58/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:23,416 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:23,417 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aad3b56713e2cfed2ad3d2a0e2facd58 columnFamilyName b 2024-12-09T03:48:23,417 DEBUG [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:23,418 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] regionserver.HStore(327): Store=aad3b56713e2cfed2ad3d2a0e2facd58/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:23,418 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:23,420 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aad3b56713e2cfed2ad3d2a0e2facd58 columnFamilyName c 2024-12-09T03:48:23,420 DEBUG [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:23,420 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] regionserver.HStore(327): Store=aad3b56713e2cfed2ad3d2a0e2facd58/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:23,420 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:23,421 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:23,421 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:23,423 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:23,423 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:23,423 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:23,424 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:23,427 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:48:23,427 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened aad3b56713e2cfed2ad3d2a0e2facd58; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71704184, jitterRate=0.06847560405731201}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:23,428 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for aad3b56713e2cfed2ad3d2a0e2facd58: Writing region info on filesystem at 1733716103412Initializing all the Stores at 1733716103413 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716103413Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716103413Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716103413Cleaning up temporary data from old regions at 1733716103423 (+10 ms)Region opened successfully at 1733716103428 (+5 ms) 2024-12-09T03:48:23,428 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing aad3b56713e2cfed2ad3d2a0e2facd58, disabling compactions & flushes 2024-12-09T03:48:23,428 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733716103347.aad3b56713e2cfed2ad3d2a0e2facd58. 2024-12-09T03:48:23,428 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733716103347.aad3b56713e2cfed2ad3d2a0e2facd58. 2024-12-09T03:48:23,429 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733716103347.aad3b56713e2cfed2ad3d2a0e2facd58. after waiting 0 ms 2024-12-09T03:48:23,429 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733716103347.aad3b56713e2cfed2ad3d2a0e2facd58. 2024-12-09T03:48:23,429 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733716103347.aad3b56713e2cfed2ad3d2a0e2facd58. 2024-12-09T03:48:23,429 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for aad3b56713e2cfed2ad3d2a0e2facd58: Waiting for close lock at 1733716103428Disabling compacts and flushes for region at 1733716103428Disabling writes for close at 1733716103429 (+1 ms)Writing region close event to WAL at 1733716103429Closed at 1733716103429 2024-12-09T03:48:23,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741864_1041 (size=175) 2024-12-09T03:48:23,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741864_1041 (size=175) 2024-12-09T03:48:23,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741864_1041 (size=175) 2024-12-09T03:48:23,436 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T03:48:23,436 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-93977570:(num 1733716103353) 2024-12-09T03:48:23,436 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:23,438 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:23,453 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103438, exclude list is [], retry=0 2024-12-09T03:48:23,456 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:23,457 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:23,457 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:23,485 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103438 2024-12-09T03:48:23,485 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:36521:36521)] 2024-12-09T03:48:23,611 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T03:48:23,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741866_1043 (size=371388) 2024-12-09T03:48:23,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741866_1043 (size=371388) 2024-12-09T03:48:23,612 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T03:48:23,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741866_1043 (size=371388) 2024-12-09T03:48:23,614 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T03:48:23,614 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T03:48:23,615 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T03:48:23,615 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T03:48:23,616 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-09T03:48:23,616 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-09T03:48:23,634 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103438, size=362.7 K (371388bytes) 2024-12-09T03:48:23,634 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103438 2024-12-09T03:48:23,635 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103438 after 1ms 2024-12-09T03:48:23,665 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:23,667 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103438 took 33ms 2024-12-09T03:48:23,675 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733716103438.temp 2024-12-09T03:48:23,702 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103438 so closing down 2024-12-09T03:48:23,702 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T03:48:23,706 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000000001-wal.1733716103438.temp 2024-12-09T03:48:23,723 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T03:48:23,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741867_1044 (size=371388) 2024-12-09T03:48:23,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741867_1044 (size=371388) 2024-12-09T03:48:23,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741867_1044 (size=371388) 2024-12-09T03:48:23,727 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000000001-wal.1733716103438.temp (wrote 3000 edits, skipped 0 edits in 16 ms) 2024-12-09T03:48:23,729 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000000001-wal.1733716103438.temp to hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000003000 2024-12-09T03:48:23,729 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 62 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103438, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-12-09T03:48:23,729 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103438, journal: Splitting hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103438, size=362.7 K (371388bytes) at 1733716103634Split 1024 edits, skipped 0 edits. at 1733716103679 (+45 ms)Split 2048 edits, skipped 0 edits. at 1733716103690 (+11 ms)Finishing writing output for hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103438 so closing down at 1733716103702 (+12 ms)Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000000001-wal.1733716103438.temp at 1733716103706 (+4 ms)3 split writer threads finished at 1733716103723 (+17 ms)Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000000001-wal.1733716103438.temp (wrote 3000 edits, skipped 0 edits in 16 ms) at 1733716103727 (+4 ms)Rename recovered edits hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000000001-wal.1733716103438.temp to hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000003000 at 1733716103729 (+2 ms)Processed 3000 edits across 1 Regions in 62 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103438, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1733716103729 2024-12-09T03:48:23,731 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103438 to hdfs://localhost:35437/hbase/oldWALs/wal.1733716103438 2024-12-09T03:48:23,732 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000003000 2024-12-09T03:48:23,732 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:23,734 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:23,749 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103735, exclude list is [], retry=0 2024-12-09T03:48:23,753 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:23,754 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:23,754 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:23,784 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103735 2024-12-09T03:48:23,785 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:37281:37281)] 2024-12-09T03:48:23,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741868_1045 (size=371515) 2024-12-09T03:48:23,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741868_1045 (size=371515) 2024-12-09T03:48:23,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741868_1045 (size=371515) 2024-12-09T03:48:23,959 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103735, size=362.8 K (371515bytes) 2024-12-09T03:48:23,959 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103735 2024-12-09T03:48:23,959 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103735 after 0ms 2024-12-09T03:48:24,004 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:24,006 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103735 took 48ms 2024-12-09T03:48:24,011 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733716103735.temp 2024-12-09T03:48:24,057 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103735 so closing down 2024-12-09T03:48:24,057 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T03:48:24,060 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000003001-wal.1733716103735.temp 2024-12-09T03:48:24,078 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T03:48:24,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741869_1046 (size=371515) 2024-12-09T03:48:24,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741869_1046 (size=371515) 2024-12-09T03:48:24,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741869_1046 (size=371515) 2024-12-09T03:48:24,085 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000003001-wal.1733716103735.temp (wrote 3000 edits, skipped 0 edits in 17 ms) 2024-12-09T03:48:24,086 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000003001-wal.1733716103735.temp to hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000006000 2024-12-09T03:48:24,087 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 81 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103735, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-12-09T03:48:24,087 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103735, journal: Splitting hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103735, size=362.8 K (371515bytes) at 1733716103959Split 1024 edits, skipped 0 edits. at 1733716104025 (+66 ms)Split 2048 edits, skipped 0 edits. at 1733716104042 (+17 ms)Finishing writing output for hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103735 so closing down at 1733716104057 (+15 ms)Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000003001-wal.1733716103735.temp at 1733716104060 (+3 ms)3 split writer threads finished at 1733716104078 (+18 ms)Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000003001-wal.1733716103735.temp (wrote 3000 edits, skipped 0 edits in 17 ms) at 1733716104085 (+7 ms)Rename recovered edits hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000003001-wal.1733716103735.temp to hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000006000 at 1733716104086 (+1 ms)Processed 3000 edits across 1 Regions in 81 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103735, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1733716104087 (+1 ms) 2024-12-09T03:48:24,089 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716103735 to hdfs://localhost:35437/hbase/oldWALs/wal.1733716103735 2024-12-09T03:48:24,090 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000006000 2024-12-09T03:48:24,090 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:24,092 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/test2727-manual,16010,1733716103345, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:24,107 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716104093, exclude list is [], retry=0 2024-12-09T03:48:24,110 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:24,110 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:24,111 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:24,142 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733716103345/wal.1733716104093 2024-12-09T03:48:24,142 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:38745:38745)] 2024-12-09T03:48:24,142 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => aad3b56713e2cfed2ad3d2a0e2facd58, NAME => 'test2727,,1733716103347.aad3b56713e2cfed2ad3d2a0e2facd58.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:24,142 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733716103347.aad3b56713e2cfed2ad3d2a0e2facd58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:24,143 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:24,143 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:24,144 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:24,146 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aad3b56713e2cfed2ad3d2a0e2facd58 columnFamilyName a 2024-12-09T03:48:24,146 DEBUG [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:24,146 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] regionserver.HStore(327): Store=aad3b56713e2cfed2ad3d2a0e2facd58/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:24,146 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:24,147 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aad3b56713e2cfed2ad3d2a0e2facd58 columnFamilyName b 2024-12-09T03:48:24,147 DEBUG [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:24,148 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] regionserver.HStore(327): Store=aad3b56713e2cfed2ad3d2a0e2facd58/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:24,148 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:24,149 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aad3b56713e2cfed2ad3d2a0e2facd58 columnFamilyName c 2024-12-09T03:48:24,149 DEBUG [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:24,149 INFO [StoreOpener-aad3b56713e2cfed2ad3d2a0e2facd58-1 {}] regionserver.HStore(327): Store=aad3b56713e2cfed2ad3d2a0e2facd58/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:24,150 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:24,150 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:24,153 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:24,154 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000003000 2024-12-09T03:48:24,185 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:24,220 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000003000 2024-12-09T03:48:24,221 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000006000 2024-12-09T03:48:24,253 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:24,312 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000006000 2024-12-09T03:48:24,313 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing aad3b56713e2cfed2ad3d2a0e2facd58 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-09T03:48:24,333 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/.tmp/a/85cb37fa9a0c458f8a0a6eeb8ff34376 is 41, key is test2727/a:100/1733716103790/Put/seqid=0 2024-12-09T03:48:24,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741871_1048 (size=84227) 2024-12-09T03:48:24,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741871_1048 (size=84227) 2024-12-09T03:48:24,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741871_1048 (size=84227) 2024-12-09T03:48:24,342 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/.tmp/a/85cb37fa9a0c458f8a0a6eeb8ff34376 2024-12-09T03:48:24,372 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/.tmp/b/ca2e3fdf88254ff69fd6ac775eabe9c3 is 41, key is test2727/b:100/1733716103838/Put/seqid=0 2024-12-09T03:48:24,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741872_1049 (size=84609) 2024-12-09T03:48:24,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741872_1049 (size=84609) 2024-12-09T03:48:24,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741872_1049 (size=84609) 2024-12-09T03:48:24,380 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/.tmp/b/ca2e3fdf88254ff69fd6ac775eabe9c3 2024-12-09T03:48:24,406 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/.tmp/c/aaa4ce55807c40958163fb652b5e66bf is 41, key is test2727/c:100/1733716103885/Put/seqid=0 2024-12-09T03:48:24,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741873_1050 (size=84609) 2024-12-09T03:48:24,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741873_1050 (size=84609) 2024-12-09T03:48:24,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741873_1050 (size=84609) 2024-12-09T03:48:24,413 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/.tmp/c/aaa4ce55807c40958163fb652b5e66bf 2024-12-09T03:48:24,420 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/.tmp/a/85cb37fa9a0c458f8a0a6eeb8ff34376 as hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/a/85cb37fa9a0c458f8a0a6eeb8ff34376 2024-12-09T03:48:24,426 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/a/85cb37fa9a0c458f8a0a6eeb8ff34376, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-09T03:48:24,428 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/.tmp/b/ca2e3fdf88254ff69fd6ac775eabe9c3 as hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/b/ca2e3fdf88254ff69fd6ac775eabe9c3 2024-12-09T03:48:24,434 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/b/ca2e3fdf88254ff69fd6ac775eabe9c3, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-09T03:48:24,435 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/.tmp/c/aaa4ce55807c40958163fb652b5e66bf as hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/c/aaa4ce55807c40958163fb652b5e66bf 2024-12-09T03:48:24,441 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/c/aaa4ce55807c40958163fb652b5e66bf, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-09T03:48:24,442 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for aad3b56713e2cfed2ad3d2a0e2facd58 in 130ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-09T03:48:24,442 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000003000 2024-12-09T03:48:24,443 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/0000000000000006000 2024-12-09T03:48:24,444 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:24,444 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:24,445 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:24,446 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for aad3b56713e2cfed2ad3d2a0e2facd58 2024-12-09T03:48:24,449 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/test2727/aad3b56713e2cfed2ad3d2a0e2facd58/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-09T03:48:24,450 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened aad3b56713e2cfed2ad3d2a0e2facd58; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72499235, jitterRate=0.08032278716564178}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:24,451 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for aad3b56713e2cfed2ad3d2a0e2facd58: Writing region info on filesystem at 1733716104143Initializing all the Stores at 1733716104144 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716104144Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716104144Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716104144Obtaining lock to block concurrent updates at 1733716104313 (+169 ms)Preparing flush snapshotting stores in aad3b56713e2cfed2ad3d2a0e2facd58 at 1733716104313Finished memstore snapshotting test2727,,1733716103347.aad3b56713e2cfed2ad3d2a0e2facd58., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733716104313Flushing stores of test2727,,1733716103347.aad3b56713e2cfed2ad3d2a0e2facd58. at 1733716104313Flushing aad3b56713e2cfed2ad3d2a0e2facd58/a: creating writer at 1733716104313Flushing aad3b56713e2cfed2ad3d2a0e2facd58/a: appending metadata at 1733716104333 (+20 ms)Flushing aad3b56713e2cfed2ad3d2a0e2facd58/a: closing flushed file at 1733716104333Flushing aad3b56713e2cfed2ad3d2a0e2facd58/b: creating writer at 1733716104349 (+16 ms)Flushing aad3b56713e2cfed2ad3d2a0e2facd58/b: appending metadata at 1733716104371 (+22 ms)Flushing aad3b56713e2cfed2ad3d2a0e2facd58/b: closing flushed file at 1733716104371Flushing aad3b56713e2cfed2ad3d2a0e2facd58/c: creating writer at 1733716104386 (+15 ms)Flushing aad3b56713e2cfed2ad3d2a0e2facd58/c: appending metadata at 1733716104405 (+19 ms)Flushing aad3b56713e2cfed2ad3d2a0e2facd58/c: closing flushed file at 1733716104405Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f35120d: reopening flushed file at 1733716104419 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d64896d: reopening flushed file at 1733716104427 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2644b5f1: reopening flushed file at 1733716104434 (+7 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for aad3b56713e2cfed2ad3d2a0e2facd58 in 130ms, sequenceid=6000, compaction requested=false; wal=null at 1733716104442 (+8 ms)Cleaning up temporary data from old regions at 1733716104444 (+2 ms)Region opened successfully at 1733716104451 (+7 ms) 2024-12-09T03:48:24,452 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-09T03:48:24,452 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing aad3b56713e2cfed2ad3d2a0e2facd58, disabling compactions & flushes 2024-12-09T03:48:24,452 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733716103347.aad3b56713e2cfed2ad3d2a0e2facd58. 2024-12-09T03:48:24,452 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733716103347.aad3b56713e2cfed2ad3d2a0e2facd58. 2024-12-09T03:48:24,452 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733716103347.aad3b56713e2cfed2ad3d2a0e2facd58. after waiting 0 ms 2024-12-09T03:48:24,452 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733716103347.aad3b56713e2cfed2ad3d2a0e2facd58. 2024-12-09T03:48:24,454 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733716103347.aad3b56713e2cfed2ad3d2a0e2facd58. 2024-12-09T03:48:24,454 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for aad3b56713e2cfed2ad3d2a0e2facd58: Waiting for close lock at 1733716104452Disabling compacts and flushes for region at 1733716104452Disabling writes for close at 1733716104452Writing region close event to WAL at 1733716104454 (+2 ms)Closed at 1733716104454 2024-12-09T03:48:24,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741870_1047 (size=175) 2024-12-09T03:48:24,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741870_1047 (size=175) 2024-12-09T03:48:24,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741870_1047 (size=175) 2024-12-09T03:48:24,460 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T03:48:24,460 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733716104093) 2024-12-09T03:48:24,475 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=401 (was 395) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:48476 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:39988 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:51528 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:39970 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=929 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=222 (was 222), ProcessCount=11 (was 11), AvailableMemoryMB=11728 (was 11969) 2024-12-09T03:48:24,488 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=401, OpenFileDescriptor=929, MaxFileDescriptor=1048576, SystemLoadAverage=222, ProcessCount=11, AvailableMemoryMB=11728 2024-12-09T03:48:24,503 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:24,511 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733716104502, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:24,511 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733716104511 2024-12-09T03:48:24,547 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733716104502/wal.1733716104511 2024-12-09T03:48:24,550 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:38745:38745)] 2024-12-09T03:48:24,553 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 55d6e34187c865bf6a864e37cff3ca1b, NAME => 'testSequentialEditLogSeqNum,,1733716104504.55d6e34187c865bf6a864e37cff3ca1b.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:24,554 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733716104504.55d6e34187c865bf6a864e37cff3ca1b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:24,554 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 55d6e34187c865bf6a864e37cff3ca1b 2024-12-09T03:48:24,554 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 55d6e34187c865bf6a864e37cff3ca1b 2024-12-09T03:48:24,555 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:35437/hbase/data/default/testSequentialEditLogSeqNum/55d6e34187c865bf6a864e37cff3ca1b doesn't exist for region: 55d6e34187c865bf6a864e37cff3ca1b on table testSequentialEditLogSeqNum 2024-12-09T03:48:24,555 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 55d6e34187c865bf6a864e37cff3ca1b on table testSequentialEditLogSeqNum 2024-12-09T03:48:24,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741875_1052 (size=62) 2024-12-09T03:48:24,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741875_1052 (size=62) 2024-12-09T03:48:24,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741875_1052 (size=62) 2024-12-09T03:48:24,575 INFO [StoreOpener-55d6e34187c865bf6a864e37cff3ca1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 55d6e34187c865bf6a864e37cff3ca1b 2024-12-09T03:48:24,577 INFO [StoreOpener-55d6e34187c865bf6a864e37cff3ca1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55d6e34187c865bf6a864e37cff3ca1b columnFamilyName a 2024-12-09T03:48:24,578 DEBUG [StoreOpener-55d6e34187c865bf6a864e37cff3ca1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:24,578 INFO [StoreOpener-55d6e34187c865bf6a864e37cff3ca1b-1 {}] regionserver.HStore(327): Store=55d6e34187c865bf6a864e37cff3ca1b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:24,578 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 55d6e34187c865bf6a864e37cff3ca1b 2024-12-09T03:48:24,579 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testSequentialEditLogSeqNum/55d6e34187c865bf6a864e37cff3ca1b 2024-12-09T03:48:24,579 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testSequentialEditLogSeqNum/55d6e34187c865bf6a864e37cff3ca1b 2024-12-09T03:48:24,580 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 55d6e34187c865bf6a864e37cff3ca1b 2024-12-09T03:48:24,580 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 55d6e34187c865bf6a864e37cff3ca1b 2024-12-09T03:48:24,582 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 55d6e34187c865bf6a864e37cff3ca1b 2024-12-09T03:48:24,584 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testSequentialEditLogSeqNum/55d6e34187c865bf6a864e37cff3ca1b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:48:24,584 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 55d6e34187c865bf6a864e37cff3ca1b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72276854, jitterRate=0.07700905203819275}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T03:48:24,585 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 55d6e34187c865bf6a864e37cff3ca1b: Writing region info on filesystem at 1733716104554Initializing all the Stores at 1733716104571 (+17 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716104571Cleaning up temporary data from old regions at 1733716104580 (+9 ms)Region opened successfully at 1733716104585 (+5 ms) 2024-12-09T03:48:24,596 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 55d6e34187c865bf6a864e37cff3ca1b 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-09T03:48:24,618 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testSequentialEditLogSeqNum/55d6e34187c865bf6a864e37cff3ca1b/.tmp/a/7a89bf84d43e4acea6a8a6b31bb581e3 is 81, key is testSequentialEditLogSeqNum/a:x0/1733716104585/Put/seqid=0 2024-12-09T03:48:24,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741876_1053 (size=5833) 2024-12-09T03:48:24,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741876_1053 (size=5833) 2024-12-09T03:48:24,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741876_1053 (size=5833) 2024-12-09T03:48:24,626 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testSequentialEditLogSeqNum/55d6e34187c865bf6a864e37cff3ca1b/.tmp/a/7a89bf84d43e4acea6a8a6b31bb581e3 2024-12-09T03:48:24,633 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testSequentialEditLogSeqNum/55d6e34187c865bf6a864e37cff3ca1b/.tmp/a/7a89bf84d43e4acea6a8a6b31bb581e3 as hdfs://localhost:35437/hbase/data/default/testSequentialEditLogSeqNum/55d6e34187c865bf6a864e37cff3ca1b/a/7a89bf84d43e4acea6a8a6b31bb581e3 2024-12-09T03:48:24,640 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testSequentialEditLogSeqNum/55d6e34187c865bf6a864e37cff3ca1b/a/7a89bf84d43e4acea6a8a6b31bb581e3, entries=10, sequenceid=13, filesize=5.7 K 2024-12-09T03:48:24,641 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 55d6e34187c865bf6a864e37cff3ca1b in 45ms, sequenceid=13, compaction requested=false 2024-12-09T03:48:24,641 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 55d6e34187c865bf6a864e37cff3ca1b: 2024-12-09T03:48:24,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T03:48:24,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T03:48:24,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T03:48:24,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T03:48:24,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T03:48:24,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741874_1051 (size=3256) 2024-12-09T03:48:24,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741874_1051 (size=3256) 2024-12-09T03:48:24,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741874_1051 (size=3256) 2024-12-09T03:48:24,667 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35437/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733716104502/wal.1733716104511, size=3.2 K (3256bytes) 2024-12-09T03:48:24,667 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35437/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733716104502/wal.1733716104511 2024-12-09T03:48:24,668 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35437/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733716104502/wal.1733716104511 after 1ms 2024-12-09T03:48:24,697 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:24,697 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35437/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733716104502/wal.1733716104511 took 30ms 2024-12-09T03:48:24,700 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35437/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733716104502/wal.1733716104511 so closing down 2024-12-09T03:48:24,700 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T03:48:24,701 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733716104511.temp 2024-12-09T03:48:24,740 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testSequentialEditLogSeqNum/55d6e34187c865bf6a864e37cff3ca1b/recovered.edits/0000000000000000003-wal.1733716104511.temp 2024-12-09T03:48:24,741 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T03:48:24,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741877_1054 (size=2650) 2024-12-09T03:48:24,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741877_1054 (size=2650) 2024-12-09T03:48:24,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741877_1054 (size=2650) 2024-12-09T03:48:24,769 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testSequentialEditLogSeqNum/55d6e34187c865bf6a864e37cff3ca1b/recovered.edits/0000000000000000003-wal.1733716104511.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-09T03:48:24,773 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35437/hbase/data/default/testSequentialEditLogSeqNum/55d6e34187c865bf6a864e37cff3ca1b/recovered.edits/0000000000000000003-wal.1733716104511.temp to hdfs://localhost:35437/hbase/data/default/testSequentialEditLogSeqNum/55d6e34187c865bf6a864e37cff3ca1b/recovered.edits/0000000000000000020 2024-12-09T03:48:24,773 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 76 ms; skipped=2; WAL=hdfs://localhost:35437/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733716104502/wal.1733716104511, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-12-09T03:48:24,773 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35437/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733716104502/wal.1733716104511, journal: Splitting hdfs://localhost:35437/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733716104502/wal.1733716104511, size=3.2 K (3256bytes) at 1733716104667Finishing writing output for hdfs://localhost:35437/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733716104502/wal.1733716104511 so closing down at 1733716104700 (+33 ms)Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testSequentialEditLogSeqNum/55d6e34187c865bf6a864e37cff3ca1b/recovered.edits/0000000000000000003-wal.1733716104511.temp at 1733716104740 (+40 ms)3 split writer threads finished at 1733716104741 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testSequentialEditLogSeqNum/55d6e34187c865bf6a864e37cff3ca1b/recovered.edits/0000000000000000003-wal.1733716104511.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733716104769 (+28 ms)Rename recovered edits hdfs://localhost:35437/hbase/data/default/testSequentialEditLogSeqNum/55d6e34187c865bf6a864e37cff3ca1b/recovered.edits/0000000000000000003-wal.1733716104511.temp to hdfs://localhost:35437/hbase/data/default/testSequentialEditLogSeqNum/55d6e34187c865bf6a864e37cff3ca1b/recovered.edits/0000000000000000020 at 1733716104773 (+4 ms)Processed 17 edits across 1 Regions in 76 ms; skipped=2; WAL=hdfs://localhost:35437/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733716104502/wal.1733716104511, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1733716104773 2024-12-09T03:48:24,796 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=406 (was 401) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:48476 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:39988 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:51528 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:39970 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=967 (was 929) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=222 (was 222), ProcessCount=11 (was 11), AvailableMemoryMB=11722 (was 11728) 2024-12-09T03:48:24,813 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=406, OpenFileDescriptor=967, MaxFileDescriptor=1048576, SystemLoadAverage=222, ProcessCount=11, AvailableMemoryMB=11725 2024-12-09T03:48:24,841 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:24,844 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:24,877 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:48:24,881 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-11580337, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/hregion-11580337, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:24,903 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-11580337/hregion-11580337.1733716104882, exclude list is [], retry=0 2024-12-09T03:48:24,909 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:24,912 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:24,914 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:24,948 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-11580337/hregion-11580337.1733716104882 2024-12-09T03:48:24,948 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:37281:37281)] 2024-12-09T03:48:24,948 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 181989cad929c353c80e042a5da3d6bb, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35437/hbase 2024-12-09T03:48:24,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741879_1056 (size=70) 2024-12-09T03:48:24,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741879_1056 (size=70) 2024-12-09T03:48:24,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741879_1056 (size=70) 2024-12-09T03:48:24,968 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:24,970 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:24,972 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 181989cad929c353c80e042a5da3d6bb columnFamilyName a 2024-12-09T03:48:24,972 DEBUG [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:24,973 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(327): Store=181989cad929c353c80e042a5da3d6bb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:24,973 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:24,975 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 181989cad929c353c80e042a5da3d6bb columnFamilyName b 2024-12-09T03:48:24,975 DEBUG [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:24,975 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(327): Store=181989cad929c353c80e042a5da3d6bb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:24,976 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:24,977 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 181989cad929c353c80e042a5da3d6bb columnFamilyName c 2024-12-09T03:48:24,977 DEBUG [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:24,978 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(327): Store=181989cad929c353c80e042a5da3d6bb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:24,978 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:24,979 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:24,979 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:24,981 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:24,981 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:24,981 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:24,983 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:24,985 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:48:24,986 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 181989cad929c353c80e042a5da3d6bb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70709901, jitterRate=0.05365963280200958}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:24,986 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 181989cad929c353c80e042a5da3d6bb: Writing region info on filesystem at 1733716104969Initializing all the Stores at 1733716104970 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716104970Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716104970Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716104970Cleaning up temporary data from old regions at 1733716104981 (+11 ms)Region opened successfully at 1733716104986 (+5 ms) 2024-12-09T03:48:24,987 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 181989cad929c353c80e042a5da3d6bb, disabling compactions & flushes 2024-12-09T03:48:24,987 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb. 2024-12-09T03:48:24,987 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb. 2024-12-09T03:48:24,987 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb. after waiting 0 ms 2024-12-09T03:48:24,987 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb. 2024-12-09T03:48:24,987 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb. 2024-12-09T03:48:24,987 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 181989cad929c353c80e042a5da3d6bb: Waiting for close lock at 1733716104987Disabling compacts and flushes for region at 1733716104987Disabling writes for close at 1733716104987Writing region close event to WAL at 1733716104987Closed at 1733716104987 2024-12-09T03:48:24,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741878_1055 (size=175) 2024-12-09T03:48:24,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741878_1055 (size=175) 2024-12-09T03:48:24,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741878_1055 (size=175) 2024-12-09T03:48:24,999 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T03:48:24,999 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-11580337:(num 1733716104882) 2024-12-09T03:48:24,999 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:25,001 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:25,016 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002, exclude list is [], retry=0 2024-12-09T03:48:25,019 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:25,019 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:25,020 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:25,056 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002 2024-12-09T03:48:25,057 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:37281:37281)] 2024-12-09T03:48:25,057 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 181989cad929c353c80e042a5da3d6bb, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:25,057 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:25,058 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:25,058 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:25,060 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:25,061 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 181989cad929c353c80e042a5da3d6bb columnFamilyName a 2024-12-09T03:48:25,061 DEBUG [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:25,062 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(327): Store=181989cad929c353c80e042a5da3d6bb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:25,062 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:25,063 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 181989cad929c353c80e042a5da3d6bb columnFamilyName b 2024-12-09T03:48:25,063 DEBUG [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:25,064 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(327): Store=181989cad929c353c80e042a5da3d6bb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:25,064 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:25,065 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 181989cad929c353c80e042a5da3d6bb columnFamilyName c 2024-12-09T03:48:25,065 DEBUG [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:25,065 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(327): Store=181989cad929c353c80e042a5da3d6bb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:25,065 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:25,066 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:25,067 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:25,069 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:25,069 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:25,069 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:25,070 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:25,071 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 181989cad929c353c80e042a5da3d6bb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72280152, jitterRate=0.07705819606781006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:25,072 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 181989cad929c353c80e042a5da3d6bb: Writing region info on filesystem at 1733716105058Initializing all the Stores at 1733716105059 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716105059Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716105060 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716105060Cleaning up temporary data from old regions at 1733716105069 (+9 ms)Region opened successfully at 1733716105072 (+3 ms) 2024-12-09T03:48:25,076 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733716105075/Put/seqid=0 2024-12-09T03:48:25,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741881_1058 (size=4826) 2024-12-09T03:48:25,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741881_1058 (size=4826) 2024-12-09T03:48:25,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741881_1058 (size=4826) 2024-12-09T03:48:25,089 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35437/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 181989cad929c353c80e042a5da3d6bb/a 2024-12-09T03:48:25,098 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-09T03:48:25,098 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T03:48:25,098 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 181989cad929c353c80e042a5da3d6bb: 2024-12-09T03:48:25,100 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/a/da1422c33de64d208e9792e5cbace76b_SeqId_3_ 2024-12-09T03:48:25,101 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35437/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 181989cad929c353c80e042a5da3d6bb/a as hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/a/da1422c33de64d208e9792e5cbace76b_SeqId_3_ - updating store file list. 2024-12-09T03:48:25,106 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for da1422c33de64d208e9792e5cbace76b_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-09T03:48:25,106 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/a/da1422c33de64d208e9792e5cbace76b_SeqId_3_ into 181989cad929c353c80e042a5da3d6bb/a 2024-12-09T03:48:25,106 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35437/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 181989cad929c353c80e042a5da3d6bb/a (new location: hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/a/da1422c33de64d208e9792e5cbace76b_SeqId_3_) 2024-12-09T03:48:25,150 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35437/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002, size=0 (0bytes) 2024-12-09T03:48:25,150 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35437/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002 might be still open, length is 0 2024-12-09T03:48:25,150 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35437/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002 2024-12-09T03:48:25,150 WARN [IPC Server handler 2 on default port 35437 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-09T03:48:25,151 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35437/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002 after 1ms 2024-12-09T03:48:27,123 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:48598 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39115:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48598 dst: /127.0.0.1:39115 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39115 remote=/127.0.0.1:48598]. Total timeout mills is 60000, 57990 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:27,123 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:51596 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45815:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51596 dst: /127.0.0.1:45815 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:27,124 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:40174 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36217:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40174 dst: /127.0.0.1:36217 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:27,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741880_1059 (size=606) 2024-12-09T03:48:27,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741880_1059 (size=606) 2024-12-09T03:48:29,152 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35437/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002 after 4002ms 2024-12-09T03:48:29,181 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:29,181 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35437/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002 took 4032ms 2024-12-09T03:48:29,185 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35437/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002; continuing. 2024-12-09T03:48:29,185 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35437/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002 so closing down 2024-12-09T03:48:29,185 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T03:48:29,187 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733716105002.temp 2024-12-09T03:48:29,231 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/recovered.edits/0000000000000000005-wal.1733716105002.temp 2024-12-09T03:48:29,231 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T03:48:29,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741882_1060 (size=329) 2024-12-09T03:48:29,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741882_1060 (size=329) 2024-12-09T03:48:29,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741882_1060 (size=329) 2024-12-09T03:48:29,246 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/recovered.edits/0000000000000000005-wal.1733716105002.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T03:48:29,248 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/recovered.edits/0000000000000000005-wal.1733716105002.temp to hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/recovered.edits/0000000000000000005 2024-12-09T03:48:29,248 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 66 ms; skipped=1; WAL=hdfs://localhost:35437/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T03:48:29,248 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35437/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002, journal: Splitting hdfs://localhost:35437/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002, size=0 (0bytes) at 1733716105150Finishing writing output for hdfs://localhost:35437/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002 so closing down at 1733716109185 (+4035 ms)Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/recovered.edits/0000000000000000005-wal.1733716105002.temp at 1733716109231 (+46 ms)3 split writer threads finished at 1733716109231Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/recovered.edits/0000000000000000005-wal.1733716105002.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733716109246 (+15 ms)Rename recovered edits hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/recovered.edits/0000000000000000005-wal.1733716105002.temp to hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/recovered.edits/0000000000000000005 at 1733716109248 (+2 ms)Processed 2 edits across 1 Regions in 66 ms; skipped=1; WAL=hdfs://localhost:35437/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002, size=0, length=0, corrupted=false, cancelled=false at 1733716109248 2024-12-09T03:48:29,251 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35437/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002 to hdfs://localhost:35437/hbase/oldWALs/wal.1733716105002 2024-12-09T03:48:29,252 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/recovered.edits/0000000000000000005 2024-12-09T03:48:29,252 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:29,254 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:29,270 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716109255, exclude list is [], retry=0 2024-12-09T03:48:29,275 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:29,276 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:29,276 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:29,312 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716109255 2024-12-09T03:48:29,313 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:38745:38745)] 2024-12-09T03:48:29,313 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 181989cad929c353c80e042a5da3d6bb, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:29,314 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:29,314 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:29,314 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:29,316 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:29,318 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 181989cad929c353c80e042a5da3d6bb columnFamilyName a 2024-12-09T03:48:29,318 DEBUG [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:29,327 DEBUG [StoreFileOpener-181989cad929c353c80e042a5da3d6bb-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for da1422c33de64d208e9792e5cbace76b_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-09T03:48:29,327 DEBUG [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/a/da1422c33de64d208e9792e5cbace76b_SeqId_3_ 2024-12-09T03:48:29,327 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(327): Store=181989cad929c353c80e042a5da3d6bb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:29,327 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:29,329 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 181989cad929c353c80e042a5da3d6bb columnFamilyName b 2024-12-09T03:48:29,329 DEBUG [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:29,330 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(327): Store=181989cad929c353c80e042a5da3d6bb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:29,330 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:29,331 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 181989cad929c353c80e042a5da3d6bb columnFamilyName c 2024-12-09T03:48:29,331 DEBUG [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:29,333 INFO [StoreOpener-181989cad929c353c80e042a5da3d6bb-1 {}] regionserver.HStore(327): Store=181989cad929c353c80e042a5da3d6bb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:29,333 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:29,334 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:29,336 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:29,337 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/recovered.edits/0000000000000000005 2024-12-09T03:48:29,367 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:29,368 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/recovered.edits/0000000000000000005 2024-12-09T03:48:29,368 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 181989cad929c353c80e042a5da3d6bb 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-09T03:48:29,385 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/.tmp/a/79a8f69eaf8c45c5a452147ac3e10d14 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733716105112/Put/seqid=0 2024-12-09T03:48:29,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741884_1062 (size=5149) 2024-12-09T03:48:29,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741884_1062 (size=5149) 2024-12-09T03:48:29,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741884_1062 (size=5149) 2024-12-09T03:48:29,397 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/.tmp/a/79a8f69eaf8c45c5a452147ac3e10d14 2024-12-09T03:48:29,404 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/.tmp/a/79a8f69eaf8c45c5a452147ac3e10d14 as hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/a/79a8f69eaf8c45c5a452147ac3e10d14 2024-12-09T03:48:29,411 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/a/79a8f69eaf8c45c5a452147ac3e10d14, entries=1, sequenceid=5, filesize=5.0 K 2024-12-09T03:48:29,412 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 181989cad929c353c80e042a5da3d6bb in 44ms, sequenceid=5, compaction requested=false; wal=null 2024-12-09T03:48:29,413 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/recovered.edits/0000000000000000005 2024-12-09T03:48:29,414 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:29,414 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:29,415 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:29,417 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 181989cad929c353c80e042a5da3d6bb 2024-12-09T03:48:29,421 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/181989cad929c353c80e042a5da3d6bb/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-09T03:48:29,422 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 181989cad929c353c80e042a5da3d6bb; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62406480, jitterRate=-0.07007098197937012}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:29,422 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 181989cad929c353c80e042a5da3d6bb: Writing region info on filesystem at 1733716109314Initializing all the Stores at 1733716109315 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716109315Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716109316 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716109316Obtaining lock to block concurrent updates at 1733716109368 (+52 ms)Preparing flush snapshotting stores in 181989cad929c353c80e042a5da3d6bb at 1733716109368Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733716109368Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb. at 1733716109368Flushing 181989cad929c353c80e042a5da3d6bb/a: creating writer at 1733716109368Flushing 181989cad929c353c80e042a5da3d6bb/a: appending metadata at 1733716109384 (+16 ms)Flushing 181989cad929c353c80e042a5da3d6bb/a: closing flushed file at 1733716109384Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ac7abde: reopening flushed file at 1733716109403 (+19 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 181989cad929c353c80e042a5da3d6bb in 44ms, sequenceid=5, compaction requested=false; wal=null at 1733716109412 (+9 ms)Cleaning up temporary data from old regions at 1733716109414 (+2 ms)Region opened successfully at 1733716109422 (+8 ms) 2024-12-09T03:48:29,428 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 181989cad929c353c80e042a5da3d6bb, disabling compactions & flushes 2024-12-09T03:48:29,428 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb. 2024-12-09T03:48:29,428 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb. 2024-12-09T03:48:29,428 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb. after waiting 0 ms 2024-12-09T03:48:29,428 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb. 2024-12-09T03:48:29,429 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733716104842.181989cad929c353c80e042a5da3d6bb. 2024-12-09T03:48:29,429 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 181989cad929c353c80e042a5da3d6bb: Waiting for close lock at 1733716109428Disabling compacts and flushes for region at 1733716109428Disabling writes for close at 1733716109428Writing region close event to WAL at 1733716109429 (+1 ms)Closed at 1733716109429 2024-12-09T03:48:29,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741883_1061 (size=175) 2024-12-09T03:48:29,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741883_1061 (size=175) 2024-12-09T03:48:29,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741883_1061 (size=175) 2024-12-09T03:48:29,440 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T03:48:29,440 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733716109255) 2024-12-09T03:48:29,459 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=410 (was 406) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35437 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-346870569_22 at /127.0.0.1:48640 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-346870569_22 at /127.0.0.1:51638 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-346870569_22 at /127.0.0.1:40188 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:35437 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1025 (was 967) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=220 (was 222), ProcessCount=11 (was 11), AvailableMemoryMB=12446 (was 11725) - AvailableMemoryMB LEAK? - 2024-12-09T03:48:29,465 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-12-09T03:48:29,482 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=410, OpenFileDescriptor=1025, MaxFileDescriptor=1048576, SystemLoadAverage=220, ProcessCount=11, AvailableMemoryMB=12441 2024-12-09T03:48:29,482 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-12-09T03:48:29,506 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:29,513 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T03:48:29,519 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 55d0183f16d2,34431,1733716093194 2024-12-09T03:48:29,522 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1febc197 2024-12-09T03:48:29,524 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T03:48:29,528 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T03:48:29,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T03:48:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-09T03:48:29,543 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T03:48:29,546 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-09T03:48:29,546 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:29,548 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T03:48:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T03:48:29,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741885_1063 (size=694) 2024-12-09T03:48:29,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741885_1063 (size=694) 2024-12-09T03:48:29,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741885_1063 (size=694) 2024-12-09T03:48:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T03:48:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T03:48:30,012 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 940a5def324c1e7ca6d1994726e09abd, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5 2024-12-09T03:48:30,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741886_1064 (size=77) 2024-12-09T03:48:30,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741886_1064 (size=77) 2024-12-09T03:48:30,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741886_1064 (size=77) 2024-12-09T03:48:30,029 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:30,029 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 940a5def324c1e7ca6d1994726e09abd, disabling compactions & flushes 2024-12-09T03:48:30,029 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:30,030 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:30,030 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. after waiting 0 ms 2024-12-09T03:48:30,030 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:30,030 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:30,030 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 940a5def324c1e7ca6d1994726e09abd: Waiting for close lock at 1733716110029Disabling compacts and flushes for region at 1733716110029Disabling writes for close at 1733716110030 (+1 ms)Writing region close event to WAL at 1733716110030Closed at 1733716110030 2024-12-09T03:48:30,032 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T03:48:30,037 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733716110032"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733716110032"}]},"ts":"1733716110032"} 2024-12-09T03:48:30,041 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T03:48:30,043 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T03:48:30,047 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733716110044"}]},"ts":"1733716110044"} 2024-12-09T03:48:30,053 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-09T03:48:30,054 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {55d0183f16d2=0} racks are {/default-rack=0} 2024-12-09T03:48:30,056 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T03:48:30,056 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T03:48:30,056 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-09T03:48:30,056 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T03:48:30,056 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T03:48:30,056 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T03:48:30,056 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T03:48:30,056 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T03:48:30,056 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T03:48:30,056 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T03:48:30,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, ASSIGN}] 2024-12-09T03:48:30,061 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, ASSIGN 2024-12-09T03:48:30,063 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, ASSIGN; state=OFFLINE, location=55d0183f16d2,45651,1733716094038; forceNewPlan=false, retain=false 2024-12-09T03:48:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T03:48:30,216 INFO [55d0183f16d2:34431 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T03:48:30,217 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=940a5def324c1e7ca6d1994726e09abd, regionState=OPENING, regionLocation=55d0183f16d2,45651,1733716094038 2024-12-09T03:48:30,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, ASSIGN because future has completed 2024-12-09T03:48:30,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,45651,1733716094038}] 2024-12-09T03:48:30,375 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T03:48:30,377 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33435, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T03:48:30,383 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:30,383 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 940a5def324c1e7ca6d1994726e09abd, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:30,384 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:30,384 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:30,384 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:30,384 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:30,387 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:30,389 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 940a5def324c1e7ca6d1994726e09abd columnFamilyName cf1 2024-12-09T03:48:30,389 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:30,390 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(327): Store=940a5def324c1e7ca6d1994726e09abd/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:30,390 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:30,392 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 940a5def324c1e7ca6d1994726e09abd columnFamilyName cf2 2024-12-09T03:48:30,392 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:30,392 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(327): Store=940a5def324c1e7ca6d1994726e09abd/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:30,393 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:30,393 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:30,394 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:30,394 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:30,394 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:30,395 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T03:48:30,397 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:30,399 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:48:30,400 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 940a5def324c1e7ca6d1994726e09abd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69869960, jitterRate=0.04114353656768799}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T03:48:30,400 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:30,401 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 940a5def324c1e7ca6d1994726e09abd: Running coprocessor pre-open hook at 1733716110385Writing region info on filesystem at 1733716110385Initializing all the Stores at 1733716110385Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716110385Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716110387 (+2 ms)Cleaning up temporary data from old regions at 1733716110394 (+7 ms)Running coprocessor post-open hooks at 1733716110400 (+6 ms)Region opened successfully at 1733716110401 (+1 ms) 2024-12-09T03:48:30,403 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., pid=6, masterSystemTime=1733716110375 2024-12-09T03:48:30,406 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:30,406 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:30,406 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=940a5def324c1e7ca6d1994726e09abd, regionState=OPEN, openSeqNum=2, regionLocation=55d0183f16d2,45651,1733716094038 2024-12-09T03:48:30,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,45651,1733716094038 because future has completed 2024-12-09T03:48:30,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T03:48:30,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,45651,1733716094038 in 190 msec 2024-12-09T03:48:30,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T03:48:30,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, ASSIGN in 357 msec 2024-12-09T03:48:30,419 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T03:48:30,420 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733716110419"}]},"ts":"1733716110419"} 2024-12-09T03:48:30,422 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-09T03:48:30,423 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T03:48:30,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 888 msec 2024-12-09T03:48:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T03:48:30,693 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-09T03:48:30,693 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-09T03:48:30,695 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T03:48:30,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-09T03:48:30,700 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T03:48:30,701 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-09T03:48:30,713 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=2] 2024-12-09T03:48:30,714 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T03:48:30,717 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T03:48:30,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=940a5def324c1e7ca6d1994726e09abd, source=55d0183f16d2,45651,1733716094038, destination=55d0183f16d2,39859,1733716093905, warming up region on 55d0183f16d2,39859,1733716093905 2024-12-09T03:48:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T03:48:30,735 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=940a5def324c1e7ca6d1994726e09abd, source=55d0183f16d2,45651,1733716094038, destination=55d0183f16d2,39859,1733716093905, running balancer 2024-12-09T03:48:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, REOPEN/MOVE 2024-12-09T03:48:30,737 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, REOPEN/MOVE 2024-12-09T03:48:30,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58781, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T03:48:30,740 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=940a5def324c1e7ca6d1994726e09abd, regionState=CLOSING, regionLocation=55d0183f16d2,45651,1733716094038 2024-12-09T03:48:30,743 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39859 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39859 {}] regionserver.HRegion(7855): Warmup {ENCODED => 940a5def324c1e7ca6d1994726e09abd, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:30,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, REOPEN/MOVE because future has completed 2024-12-09T03:48:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39859 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:30,745 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T03:48:30,745 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:30,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,45651,1733716094038}] 2024-12-09T03:48:30,746 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 940a5def324c1e7ca6d1994726e09abd columnFamilyName cf1 2024-12-09T03:48:30,746 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:30,747 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(327): Store=940a5def324c1e7ca6d1994726e09abd/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:30,747 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:30,749 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 940a5def324c1e7ca6d1994726e09abd columnFamilyName cf2 2024-12-09T03:48:30,749 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:30,751 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(327): Store=940a5def324c1e7ca6d1994726e09abd/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39859 {}] regionserver.HRegion(1722): Closing 940a5def324c1e7ca6d1994726e09abd, disabling compactions & flushes 2024-12-09T03:48:30,751 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39859 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39859 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39859 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. after waiting 0 ms 2024-12-09T03:48:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39859 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:30,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39859 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39859 {}] regionserver.HRegion(1676): Region close journal for 940a5def324c1e7ca6d1994726e09abd: Waiting for close lock at 1733716110751Disabling compacts and flushes for region at 1733716110751Disabling writes for close at 1733716110751Writing region close event to WAL at 1733716110752 (+1 ms)Closed at 1733716110752 2024-12-09T03:48:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-09T03:48:30,906 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:30,906 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T03:48:30,907 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 940a5def324c1e7ca6d1994726e09abd, disabling compactions & flushes 2024-12-09T03:48:30,907 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:30,907 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:30,908 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. after waiting 0 ms 2024-12-09T03:48:30,908 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:30,908 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 940a5def324c1e7ca6d1994726e09abd 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-09T03:48:30,930 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/.tmp/cf1/6162ac6b06094cf4b0d7954b6d103e01 is 35, key is r1/cf1:q/1733716110717/Put/seqid=0 2024-12-09T03:48:30,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741887_1065 (size=4783) 2024-12-09T03:48:30,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741887_1065 (size=4783) 2024-12-09T03:48:30,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741887_1065 (size=4783) 2024-12-09T03:48:30,949 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/.tmp/cf1/6162ac6b06094cf4b0d7954b6d103e01 2024-12-09T03:48:30,960 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/.tmp/cf1/6162ac6b06094cf4b0d7954b6d103e01 as hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/6162ac6b06094cf4b0d7954b6d103e01 2024-12-09T03:48:30,969 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/6162ac6b06094cf4b0d7954b6d103e01, entries=1, sequenceid=5, filesize=4.7 K 2024-12-09T03:48:30,970 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 940a5def324c1e7ca6d1994726e09abd in 62ms, sequenceid=5, compaction requested=false 2024-12-09T03:48:30,971 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-09T03:48:30,979 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-09T03:48:30,983 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:30,983 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 940a5def324c1e7ca6d1994726e09abd: Waiting for close lock at 1733716110907Running coprocessor pre-close hooks at 1733716110907Disabling compacts and flushes for region at 1733716110907Disabling writes for close at 1733716110908 (+1 ms)Obtaining lock to block concurrent updates at 1733716110908Preparing flush snapshotting stores in 940a5def324c1e7ca6d1994726e09abd at 1733716110908Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733716110908Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. at 1733716110909 (+1 ms)Flushing 940a5def324c1e7ca6d1994726e09abd/cf1: creating writer at 1733716110909Flushing 940a5def324c1e7ca6d1994726e09abd/cf1: appending metadata at 1733716110929 (+20 ms)Flushing 940a5def324c1e7ca6d1994726e09abd/cf1: closing flushed file at 1733716110929Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57a1ae87: reopening flushed file at 1733716110959 (+30 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 940a5def324c1e7ca6d1994726e09abd in 62ms, sequenceid=5, compaction requested=false at 1733716110970 (+11 ms)Writing region close event to WAL at 1733716110972 (+2 ms)Running coprocessor post-close hooks at 1733716110980 (+8 ms)Closed at 1733716110983 (+3 ms) 2024-12-09T03:48:30,983 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 940a5def324c1e7ca6d1994726e09abd move to 55d0183f16d2,39859,1733716093905 record at close sequenceid=5 2024-12-09T03:48:30,988 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:30,989 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=940a5def324c1e7ca6d1994726e09abd, regionState=CLOSED 2024-12-09T03:48:30,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,45651,1733716094038 because future has completed 2024-12-09T03:48:31,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T03:48:31,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,45651,1733716094038 in 253 msec 2024-12-09T03:48:31,003 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, REOPEN/MOVE; state=CLOSED, location=55d0183f16d2,39859,1733716093905; forceNewPlan=false, retain=false 2024-12-09T03:48:31,154 INFO [55d0183f16d2:34431 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T03:48:31,155 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=940a5def324c1e7ca6d1994726e09abd, regionState=OPENING, regionLocation=55d0183f16d2,39859,1733716093905 2024-12-09T03:48:31,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, REOPEN/MOVE because future has completed 2024-12-09T03:48:31,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,39859,1733716093905}] 2024-12-09T03:48:31,318 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:31,318 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 940a5def324c1e7ca6d1994726e09abd, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:31,318 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:31,318 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:31,318 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:31,319 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:31,320 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:31,321 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 940a5def324c1e7ca6d1994726e09abd columnFamilyName cf1 2024-12-09T03:48:31,321 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:31,330 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/6162ac6b06094cf4b0d7954b6d103e01 2024-12-09T03:48:31,330 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(327): Store=940a5def324c1e7ca6d1994726e09abd/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:31,331 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:31,332 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 940a5def324c1e7ca6d1994726e09abd columnFamilyName cf2 2024-12-09T03:48:31,332 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:31,333 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(327): Store=940a5def324c1e7ca6d1994726e09abd/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:31,333 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:31,334 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:31,336 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:31,337 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:31,337 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:31,338 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T03:48:31,340 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:31,341 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 940a5def324c1e7ca6d1994726e09abd; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65077406, jitterRate=-0.030271083116531372}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T03:48:31,341 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:31,342 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 940a5def324c1e7ca6d1994726e09abd: Running coprocessor pre-open hook at 1733716111319Writing region info on filesystem at 1733716111319Initializing all the Stores at 1733716111320 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716111320Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716111320Cleaning up temporary data from old regions at 1733716111337 (+17 ms)Running coprocessor post-open hooks at 1733716111341 (+4 ms)Region opened successfully at 1733716111342 (+1 ms) 2024-12-09T03:48:31,344 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., pid=9, masterSystemTime=1733716111313 2024-12-09T03:48:31,347 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:31,348 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:31,348 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=940a5def324c1e7ca6d1994726e09abd, regionState=OPEN, openSeqNum=9, regionLocation=55d0183f16d2,39859,1733716093905 2024-12-09T03:48:31,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,39859,1733716093905 because future has completed 2024-12-09T03:48:31,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-09T03:48:31,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,39859,1733716093905 in 196 msec 2024-12-09T03:48:31,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, REOPEN/MOVE in 624 msec 2024-12-09T03:48:31,395 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T03:48:31,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T03:48:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45651 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:34400 deadline: 1733716171406, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=55d0183f16d2 port=39859 startCode=1733716093905. As of locationSeqNum=5. 2024-12-09T03:48:31,416 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=55d0183f16d2 port=39859 startCode=1733716093905. As of locationSeqNum=5. 2024-12-09T03:48:31,417 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=55d0183f16d2 port=39859 startCode=1733716093905. As of locationSeqNum=5. 2024-12-09T03:48:31,417 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,39859,1733716093905, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=55d0183f16d2 port=39859 startCode=1733716093905. As of locationSeqNum=5. 2024-12-09T03:48:31,532 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T03:48:31,534 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T03:48:31,548 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 940a5def324c1e7ca6d1994726e09abd 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-09T03:48:31,584 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/.tmp/cf1/66be218378b04d15baeee2e0184e7800 is 29, key is r1/cf1:/1733716111536/DeleteFamily/seqid=0 2024-12-09T03:48:31,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741888_1066 (size=4906) 2024-12-09T03:48:31,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741888_1066 (size=4906) 2024-12-09T03:48:31,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741888_1066 (size=4906) 2024-12-09T03:48:32,004 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/.tmp/cf1/66be218378b04d15baeee2e0184e7800 2024-12-09T03:48:32,013 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 66be218378b04d15baeee2e0184e7800 2024-12-09T03:48:32,041 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/.tmp/cf2/e2232f783c20431d81f211d138eb54e3 is 29, key is r1/cf2:/1733716111536/DeleteFamily/seqid=0 2024-12-09T03:48:32,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741889_1067 (size=4906) 2024-12-09T03:48:32,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741889_1067 (size=4906) 2024-12-09T03:48:32,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741889_1067 (size=4906) 2024-12-09T03:48:32,056 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/.tmp/cf2/e2232f783c20431d81f211d138eb54e3 2024-12-09T03:48:32,064 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e2232f783c20431d81f211d138eb54e3 2024-12-09T03:48:32,065 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/.tmp/cf1/66be218378b04d15baeee2e0184e7800 as hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/66be218378b04d15baeee2e0184e7800 2024-12-09T03:48:32,072 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 66be218378b04d15baeee2e0184e7800 2024-12-09T03:48:32,072 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/66be218378b04d15baeee2e0184e7800, entries=1, sequenceid=12, filesize=4.8 K 2024-12-09T03:48:32,073 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/.tmp/cf2/e2232f783c20431d81f211d138eb54e3 as hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf2/e2232f783c20431d81f211d138eb54e3 2024-12-09T03:48:32,079 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e2232f783c20431d81f211d138eb54e3 2024-12-09T03:48:32,079 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf2/e2232f783c20431d81f211d138eb54e3, entries=1, sequenceid=12, filesize=4.8 K 2024-12-09T03:48:32,081 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 940a5def324c1e7ca6d1994726e09abd in 533ms, sequenceid=12, compaction requested=false 2024-12-09T03:48:32,081 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 940a5def324c1e7ca6d1994726e09abd: 2024-12-09T03:48:32,083 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T03:48:32,085 DEBUG [Time-limited test {}] regionserver.HStore(1541): 940a5def324c1e7ca6d1994726e09abd/cf1 is initiating major compaction (all files) 2024-12-09T03:48:32,085 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T03:48:32,085 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:32,086 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 940a5def324c1e7ca6d1994726e09abd/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,086 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/6162ac6b06094cf4b0d7954b6d103e01, hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/66be218378b04d15baeee2e0184e7800] into tmpdir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/.tmp, totalSize=9.5 K 2024-12-09T03:48:32,087 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6162ac6b06094cf4b0d7954b6d103e01, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733716110717 2024-12-09T03:48:32,088 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 66be218378b04d15baeee2e0184e7800, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-09T03:48:32,099 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 940a5def324c1e7ca6d1994726e09abd#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T03:48:32,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741890_1068 (size=4626) 2024-12-09T03:48:32,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741890_1068 (size=4626) 2024-12-09T03:48:32,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741890_1068 (size=4626) 2024-12-09T03:48:32,125 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/.tmp/cf1/322930b6d2c9423c9af12d5203daaf38 as hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/322930b6d2c9423c9af12d5203daaf38 2024-12-09T03:48:32,141 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 940a5def324c1e7ca6d1994726e09abd/cf1 of 940a5def324c1e7ca6d1994726e09abd into 322930b6d2c9423c9af12d5203daaf38(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T03:48:32,141 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 940a5def324c1e7ca6d1994726e09abd: 2024-12-09T03:48:32,141 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-09T03:48:32,141 DEBUG [Time-limited test {}] regionserver.HStore(1541): 940a5def324c1e7ca6d1994726e09abd/cf2 is initiating major compaction (all files) 2024-12-09T03:48:32,141 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T03:48:32,141 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:48:32,141 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 940a5def324c1e7ca6d1994726e09abd/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,142 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf2/e2232f783c20431d81f211d138eb54e3] into tmpdir=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/.tmp, totalSize=4.8 K 2024-12-09T03:48:32,142 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e2232f783c20431d81f211d138eb54e3, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-09T03:48:32,150 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 940a5def324c1e7ca6d1994726e09abd#cf2#compaction#17 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T03:48:32,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741891_1069 (size=4592) 2024-12-09T03:48:32,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741891_1069 (size=4592) 2024-12-09T03:48:32,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741891_1069 (size=4592) 2024-12-09T03:48:32,174 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/.tmp/cf2/7cd795225f9a4214ade3f4d3e5727404 as hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf2/7cd795225f9a4214ade3f4d3e5727404 2024-12-09T03:48:32,185 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 940a5def324c1e7ca6d1994726e09abd/cf2 of 940a5def324c1e7ca6d1994726e09abd into 7cd795225f9a4214ade3f4d3e5727404(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T03:48:32,186 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 940a5def324c1e7ca6d1994726e09abd: 2024-12-09T03:48:32,191 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=940a5def324c1e7ca6d1994726e09abd, source=55d0183f16d2,39859,1733716093905, destination=55d0183f16d2,45651,1733716094038, warming up region on 55d0183f16d2,45651,1733716094038 2024-12-09T03:48:32,192 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=940a5def324c1e7ca6d1994726e09abd, source=55d0183f16d2,39859,1733716093905, destination=55d0183f16d2,45651,1733716094038, running balancer 2024-12-09T03:48:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, REOPEN/MOVE 2024-12-09T03:48:32,194 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, REOPEN/MOVE 2024-12-09T03:48:32,195 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45651 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45651 {}] regionserver.HRegion(7855): Warmup {ENCODED => 940a5def324c1e7ca6d1994726e09abd, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45651 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:32,196 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=940a5def324c1e7ca6d1994726e09abd, regionState=CLOSING, regionLocation=55d0183f16d2,39859,1733716093905 2024-12-09T03:48:32,197 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,198 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 940a5def324c1e7ca6d1994726e09abd columnFamilyName cf1 2024-12-09T03:48:32,198 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:32,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, REOPEN/MOVE because future has completed 2024-12-09T03:48:32,201 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T03:48:32,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,39859,1733716093905}] 2024-12-09T03:48:32,210 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/322930b6d2c9423c9af12d5203daaf38 2024-12-09T03:48:32,216 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/6162ac6b06094cf4b0d7954b6d103e01 2024-12-09T03:48:32,223 INFO [StoreFileOpener-940a5def324c1e7ca6d1994726e09abd-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 66be218378b04d15baeee2e0184e7800 2024-12-09T03:48:32,223 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/66be218378b04d15baeee2e0184e7800 2024-12-09T03:48:32,223 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(327): Store=940a5def324c1e7ca6d1994726e09abd/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:32,224 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,225 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 940a5def324c1e7ca6d1994726e09abd columnFamilyName cf2 2024-12-09T03:48:32,225 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:32,239 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf2/7cd795225f9a4214ade3f4d3e5727404 2024-12-09T03:48:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-09T03:48:32,244 INFO [StoreFileOpener-940a5def324c1e7ca6d1994726e09abd-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e2232f783c20431d81f211d138eb54e3 2024-12-09T03:48:32,245 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf2/e2232f783c20431d81f211d138eb54e3 2024-12-09T03:48:32,245 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(327): Store=940a5def324c1e7ca6d1994726e09abd/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45651 {}] regionserver.HRegion(1722): Closing 940a5def324c1e7ca6d1994726e09abd, disabling compactions & flushes 2024-12-09T03:48:32,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45651 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45651 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45651 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. after waiting 0 ms 2024-12-09T03:48:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45651 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45651 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45651 {}] regionserver.HRegion(1676): Region close journal for 940a5def324c1e7ca6d1994726e09abd: Waiting for close lock at 1733716112245Disabling compacts and flushes for region at 1733716112245Disabling writes for close at 1733716112245Writing region close event to WAL at 1733716112249 (+4 ms)Closed at 1733716112249 2024-12-09T03:48:32,356 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,356 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T03:48:32,356 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 940a5def324c1e7ca6d1994726e09abd, disabling compactions & flushes 2024-12-09T03:48:32,356 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,356 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,356 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. after waiting 0 ms 2024-12-09T03:48:32,356 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,357 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/6162ac6b06094cf4b0d7954b6d103e01, hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/66be218378b04d15baeee2e0184e7800] to archive 2024-12-09T03:48:32,361 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T03:48:32,364 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/6162ac6b06094cf4b0d7954b6d103e01 to hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/6162ac6b06094cf4b0d7954b6d103e01 2024-12-09T03:48:32,366 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/66be218378b04d15baeee2e0184e7800 to hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/66be218378b04d15baeee2e0184e7800 2024-12-09T03:48:32,380 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf2/e2232f783c20431d81f211d138eb54e3] to archive 2024-12-09T03:48:32,382 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T03:48:32,384 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf2/e2232f783c20431d81f211d138eb54e3 to hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf2/e2232f783c20431d81f211d138eb54e3 2024-12-09T03:48:32,391 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-09T03:48:32,392 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,392 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 940a5def324c1e7ca6d1994726e09abd: Waiting for close lock at 1733716112356Running coprocessor pre-close hooks at 1733716112356Disabling compacts and flushes for region at 1733716112356Disabling writes for close at 1733716112356Writing region close event to WAL at 1733716112385 (+29 ms)Running coprocessor post-close hooks at 1733716112392 (+7 ms)Closed at 1733716112392 2024-12-09T03:48:32,392 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 940a5def324c1e7ca6d1994726e09abd move to 55d0183f16d2,45651,1733716094038 record at close sequenceid=12 2024-12-09T03:48:32,395 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,396 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=940a5def324c1e7ca6d1994726e09abd, regionState=CLOSED 2024-12-09T03:48:32,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,39859,1733716093905 because future has completed 2024-12-09T03:48:32,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-09T03:48:32,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,39859,1733716093905 in 199 msec 2024-12-09T03:48:32,404 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, REOPEN/MOVE; state=CLOSED, location=55d0183f16d2,45651,1733716094038; forceNewPlan=false, retain=false 2024-12-09T03:48:32,554 INFO [55d0183f16d2:34431 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T03:48:32,555 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=940a5def324c1e7ca6d1994726e09abd, regionState=OPENING, regionLocation=55d0183f16d2,45651,1733716094038 2024-12-09T03:48:32,561 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, REOPEN/MOVE because future has completed 2024-12-09T03:48:32,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,45651,1733716094038}] 2024-12-09T03:48:32,719 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,719 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 940a5def324c1e7ca6d1994726e09abd, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:32,720 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,720 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:32,720 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,720 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,722 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,723 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 940a5def324c1e7ca6d1994726e09abd columnFamilyName cf1 2024-12-09T03:48:32,723 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:32,730 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/322930b6d2c9423c9af12d5203daaf38 2024-12-09T03:48:32,730 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(327): Store=940a5def324c1e7ca6d1994726e09abd/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:32,731 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,732 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 940a5def324c1e7ca6d1994726e09abd columnFamilyName cf2 2024-12-09T03:48:32,732 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:32,741 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf2/7cd795225f9a4214ade3f4d3e5727404 2024-12-09T03:48:32,741 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(327): Store=940a5def324c1e7ca6d1994726e09abd/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:32,741 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,742 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,744 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,745 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,745 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,746 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T03:48:32,747 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,748 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 940a5def324c1e7ca6d1994726e09abd; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72534868, jitterRate=0.08085376024246216}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T03:48:32,748 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,749 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 940a5def324c1e7ca6d1994726e09abd: Running coprocessor pre-open hook at 1733716112720Writing region info on filesystem at 1733716112720Initializing all the Stores at 1733716112721 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716112721Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716112722 (+1 ms)Cleaning up temporary data from old regions at 1733716112745 (+23 ms)Running coprocessor post-open hooks at 1733716112748 (+3 ms)Region opened successfully at 1733716112749 (+1 ms) 2024-12-09T03:48:32,750 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., pid=12, masterSystemTime=1733716112715 2024-12-09T03:48:32,753 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,753 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,754 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=940a5def324c1e7ca6d1994726e09abd, regionState=OPEN, openSeqNum=18, regionLocation=55d0183f16d2,45651,1733716094038 2024-12-09T03:48:32,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,45651,1733716094038 because future has completed 2024-12-09T03:48:32,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-09T03:48:32,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,45651,1733716094038 in 196 msec 2024-12-09T03:48:32,763 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, REOPEN/MOVE in 569 msec 2024-12-09T03:48:32,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T03:48:32,799 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T03:48:32,800 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 55d0183f16d2,45651,1733716094038: testing ***** 2024-12-09T03:48:32,801 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-09T03:48:32,804 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-09T03:48:32,805 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-09T03:48:32,812 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-09T03:48:32,814 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-09T03:48:32,829 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 400372960 }, "NonHeapMemoryUsage": { "committed": 173211648, "init": 7667712, "max": -1, "used": 170611848 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "55d0183f16d2", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 11067, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 57, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 4, "ProcessCallTime_90th_percentile": 5, "ProcessCallTime_95th_percentile": 14, "ProcessCallTime_98th_percentile": 57, "ProcessCallTime_99th_percentile": 57, "ProcessCallTime_99.9th_percentile": 57, "ProcessCallTime_TimeRangeCount_0-1": 10, "ProcessCallTime_TimeRangeCount_1-3": 13, "ProcessCallTime_TimeRangeCount_3-10": 6, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 57, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 4, "TotalCallTime_90th_percentile": 6, "TotalCallTime_95th_percentile": 15, "TotalCallTime_98th_percentile": 57, "TotalCallTime_99th_percentile": 57, "TotalCallTime_99.9th_percentile": 57, "TotalCallTime_TimeRangeCount_0-1": 8, "TotalCallTime_TimeRangeCount_1-3": 14, "TotalCallTime_TimeRangeCount_3-10": 7, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 158, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 5, "ResponseSize_SizeRangeCount_100-1000": 4, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5743 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "55d0183f16d2", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:62749", "tag.serverName": "55d0183f16d2,45651,1733716094038", "tag.clusterId": "79d41f04-398a-445f-90b8-79288500355c", "tag.Context": "regionserver", "tag.Hostname": "55d0183f16d2", "regionCount": 1, "storeCount": 0, "hlogFileCount": 3, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733716094038, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 66560, "ByteBuffAllocatorHeapAllocationRatio": 0.7697053155307054, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 4, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 1, "Put_min": 3, "Put_max": 3, "Put_mean": 3, "Put_25th_percentile": 3, "Put_median": 3, "Put_75th_percentile": 3, "Put_90th_percentile": 3, "Put_95th_percentile": 3, "Put_98th_percentile": 3, "Put_99th_percentile": 3, "Put_99.9th_percentile": 3, "Put_TimeRangeCount_1-3": 1, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 4783, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 31, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 4783, "FlushOutputSize_max": 4783, "FlushOutputSize_mean": 4783, "FlushOutputSize_25th_percentile": 4783, "FlushOutputSize_median": 4783, "FlushOutputSize_75th_percentile": 4783, "FlushOutputSize_90th_percentile": 4783, "FlushOutputSize_95th_percentile": 4783, "FlushOutputSize_98th_percentile": 4783, "FlushOutputSize_99th_percentile": 4783, "FlushOutputSize_99.9th_percentile": 4783, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 31, "FlushMemstoreSize_max": 31, "FlushMemstoreSize_mean": 31, "FlushMemstoreSize_25th_percentile": 31, "FlushMemstoreSize_median": 31, "FlushMemstoreSize_75th_percentile": 31, "FlushMemstoreSize_90th_percentile": 31, "FlushMemstoreSize_95th_percentile": 31, "FlushMemstoreSize_98th_percentile": 31, "FlushMemstoreSize_99th_percentile": 31, "FlushMemstoreSize_99.9th_percentile": 31, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 62, "FlushTime_max": 62, "FlushTime_mean": 62, "FlushTime_25th_percentile": 62, "FlushTime_median": 62, "FlushTime_75th_percentile": 62, "FlushTime_90th_percentile": 62, "FlushTime_95th_percentile": 62, "FlushTime_98th_percentile": 62, "FlushTime_99th_percentile": 62, "FlushTime_99.9th_percentile": 62, "FlushTime_TimeRangeCount_30-100": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 64, "ScanSize_max": 64, "ScanSize_mean": 64, "ScanSize_25th_percentile": 64, "ScanSize_median": 64, "ScanSize_75th_percentile": 64, "ScanSize_90th_percentile": 64, "ScanSize_95th_percentile": 64, "ScanSize_98th_percentile": 64, "ScanSize_99th_percentile": 64, "ScanSize_99.9th_percentile": 64, "ScanSize_SizeRangeCount_10-100": 1, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-09T03:48:32,833 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34431 {}] master.MasterRpcServices(700): 55d0183f16d2,45651,1733716094038 reported a fatal error: ***** ABORTING region server 55d0183f16d2,45651,1733716094038: testing ***** 2024-12-09T03:48:32,837 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '55d0183f16d2,45651,1733716094038' ***** 2024-12-09T03:48:32,837 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-09T03:48:32,837 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T03:48:32,837 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T03:48:32,838 INFO [RS:1;55d0183f16d2:45651 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-09T03:48:32,838 INFO [RS:1;55d0183f16d2:45651 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-09T03:48:32,838 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(3091): Received CLOSE for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,838 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(956): aborting server 55d0183f16d2,45651,1733716094038 2024-12-09T03:48:32,838 INFO [RS:1;55d0183f16d2:45651 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T03:48:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39859 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:37394 deadline: 1733716172838, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=55d0183f16d2 port=45651 startCode=1733716094038. As of locationSeqNum=12. 2024-12-09T03:48:32,839 INFO [RS:1;55d0183f16d2:45651 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;55d0183f16d2:45651. 2024-12-09T03:48:32,839 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 940a5def324c1e7ca6d1994726e09abd, disabling compactions & flushes 2024-12-09T03:48:32,839 DEBUG [RS:1;55d0183f16d2:45651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T03:48:32,839 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,839 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,839 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. after waiting 0 ms 2024-12-09T03:48:32,839 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,839 DEBUG [RS:1;55d0183f16d2:45651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:48:32,839 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,39859,1733716093905, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,39859,1733716093905, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=55d0183f16d2 port=45651 startCode=1733716094038. As of locationSeqNum=12. 2024-12-09T03:48:32,840 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,39859,1733716093905, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=55d0183f16d2 port=45651 startCode=1733716094038. As of locationSeqNum=12. 2024-12-09T03:48:32,840 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T03:48:32,840 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,39859,1733716093905, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=55d0183f16d2 port=45651 startCode=1733716094038. As of locationSeqNum=12. 2024-12-09T03:48:32,840 DEBUG [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(1325): Online Regions={940a5def324c1e7ca6d1994726e09abd=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.} 2024-12-09T03:48:32,840 DEBUG [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(1351): Waiting on 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:32,842 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,843 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 940a5def324c1e7ca6d1994726e09abd: Waiting for close lock at 1733716112838Running coprocessor pre-close hooks at 1733716112839 (+1 ms)Disabling compacts and flushes for region at 1733716112839Disabling writes for close at 1733716112839Writing region close event to WAL at 1733716112842 (+3 ms)Running coprocessor post-close hooks at 1733716112842Closed at 1733716112842 2024-12-09T03:48:32,843 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:32,890 INFO [regionserver/55d0183f16d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T03:48:32,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 55d0183f16d2,45651,1733716094038 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:?] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:?] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:?] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:32,954 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 55d0183f16d2,45651,1733716094038 aborting 2024-12-09T03:48:32,954 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 55d0183f16d2,45651,1733716094038 aborting 2024-12-09T03:48:32,954 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=12 from cache 2024-12-09T03:48:33,040 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(976): stopping server 55d0183f16d2,45651,1733716094038; all regions closed. 2024-12-09T03:48:33,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741834_1010 (size=2288) 2024-12-09T03:48:33,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741834_1010 (size=2288) 2024-12-09T03:48:33,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741834_1010 (size=2288) 2024-12-09T03:48:33,046 DEBUG [RS:1;55d0183f16d2:45651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:48:33,046 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T03:48:33,046 INFO [RS:1;55d0183f16d2:45651 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T03:48:33,046 INFO [RS:1;55d0183f16d2:45651 {}] hbase.ChoreService(370): Chore service for: regionserver/55d0183f16d2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T03:48:33,047 INFO [regionserver/55d0183f16d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T03:48:33,047 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T03:48:33,047 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T03:48:33,047 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T03:48:33,047 INFO [RS:1;55d0183f16d2:45651 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T03:48:33,047 INFO [RS:1;55d0183f16d2:45651 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45651 2024-12-09T03:48:33,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T03:48:33,058 INFO [RS:1;55d0183f16d2:45651 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T03:48:33,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45651-0x1019556106d0002, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/55d0183f16d2,45651,1733716094038 2024-12-09T03:48:33,059 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [55d0183f16d2,45651,1733716094038] 2024-12-09T03:48:33,061 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/55d0183f16d2,45651,1733716094038 already deleted, retry=false 2024-12-09T03:48:33,062 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 55d0183f16d2,45651,1733716094038 on 55d0183f16d2,34431,1733716093194 2024-12-09T03:48:33,067 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 55d0183f16d2,45651,1733716094038, splitWal=true, meta=false 2024-12-09T03:48:33,069 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 55d0183f16d2,45651,1733716094038 (carryingMeta=false) 55d0183f16d2,45651,1733716094038/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4a404498[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-09T03:48:33,069 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 55d0183f16d2,45651,1733716094038, splitWal=true, meta=false 2024-12-09T03:48:33,073 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 55d0183f16d2,45651,1733716094038 had 1 regions 2024-12-09T03:48:33,074 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 55d0183f16d2,45651,1733716094038, splitWal=true, meta=false, isMeta: false 2024-12-09T03:48:33,076 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038-splitting 2024-12-09T03:48:33,078 INFO [PEWorker-1 {}] master.SplitWALManager(105): 55d0183f16d2,45651,1733716094038 WAL count=1, meta=false 2024-12-09T03:48:33,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 55d0183f16d2%2C45651%2C1733716094038.1733716095554}] 2024-12-09T03:48:33,087 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=55d0183f16d2,35501,1733716094084 2024-12-09T03:48:33,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 55d0183f16d2%2C45651%2C1733716094038.1733716095554, worker=55d0183f16d2,35501,1733716094084}] 2024-12-09T03:48:33,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45651-0x1019556106d0002, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:48:33,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45651-0x1019556106d0002, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:48:33,161 INFO [RS:1;55d0183f16d2:45651 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T03:48:33,162 INFO [RS:1;55d0183f16d2:45651 {}] regionserver.HRegionServer(1031): Exiting; stopping=55d0183f16d2,45651,1733716094038; zookeeper connection closed. 2024-12-09T03:48:33,162 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4d1deadc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4d1deadc 2024-12-09T03:48:33,165 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=18] 2024-12-09T03:48:33,167 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 55d0183f16d2:45651 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 55d0183f16d2/172.17.0.2:45651 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:33,168 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=18, error=java.net.ConnectException: Call to address=55d0183f16d2:45651 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 55d0183f16d2/172.17.0.2:45651 2024-12-09T03:48:33,168 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-09T03:48:33,168 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=18 from cache 2024-12-09T03:48:33,168 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 55d0183f16d2:45651 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 55d0183f16d2/172.17.0.2:45651 2024-12-09T03:48:33,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35501 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-09T03:48:33,276 INFO [RS_LOG_REPLAY_OPS-regionserver/55d0183f16d2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038-splitting/55d0183f16d2%2C45651%2C1733716094038.1733716095554, size=2.2 K (2288bytes) 2024-12-09T03:48:33,276 INFO [RS_LOG_REPLAY_OPS-regionserver/55d0183f16d2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038-splitting/55d0183f16d2%2C45651%2C1733716094038.1733716095554 2024-12-09T03:48:33,276 INFO [RS_LOG_REPLAY_OPS-regionserver/55d0183f16d2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038-splitting/55d0183f16d2%2C45651%2C1733716094038.1733716095554 after 0ms 2024-12-09T03:48:33,306 DEBUG [RS_LOG_REPLAY_OPS-regionserver/55d0183f16d2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:33,306 INFO [RS_LOG_REPLAY_OPS-regionserver/55d0183f16d2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038-splitting/55d0183f16d2%2C45651%2C1733716094038.1733716095554 took 31ms 2024-12-09T03:48:33,316 DEBUG [RS_LOG_REPLAY_OPS-regionserver/55d0183f16d2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 940a5def324c1e7ca6d1994726e09abd: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-09T03:48:33,316 DEBUG [RS_LOG_REPLAY_OPS-regionserver/55d0183f16d2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038-splitting/55d0183f16d2%2C45651%2C1733716094038.1733716095554 so closing down 2024-12-09T03:48:33,316 DEBUG [RS_LOG_REPLAY_OPS-regionserver/55d0183f16d2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T03:48:33,316 INFO [RS_LOG_REPLAY_OPS-regionserver/55d0183f16d2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T03:48:33,316 INFO [RS_LOG_REPLAY_OPS-regionserver/55d0183f16d2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 10 ms; skipped=6; WAL=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038-splitting/55d0183f16d2%2C45651%2C1733716094038.1733716095554, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-12-09T03:48:33,317 DEBUG [RS_LOG_REPLAY_OPS-regionserver/55d0183f16d2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038-splitting/55d0183f16d2%2C45651%2C1733716094038.1733716095554, journal: Splitting hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038-splitting/55d0183f16d2%2C45651%2C1733716094038.1733716095554, size=2.2 K (2288bytes) at 1733716113276Finishing writing output for hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038-splitting/55d0183f16d2%2C45651%2C1733716094038.1733716095554 so closing down at 1733716113316 (+40 ms)3 split writer threads finished at 1733716113316Processed 6 edits across 0 Regions in 10 ms; skipped=6; WAL=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038-splitting/55d0183f16d2%2C45651%2C1733716094038.1733716095554, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1733716113316 2024-12-09T03:48:33,317 DEBUG [RS_LOG_REPLAY_OPS-regionserver/55d0183f16d2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038-splitting/55d0183f16d2%2C45651%2C1733716094038.1733716095554 2024-12-09T03:48:33,318 DEBUG [RS_LOG_REPLAY_OPS-regionserver/55d0183f16d2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-09T03:48:33,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34431 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-09T03:48:33,328 INFO [PEWorker-3 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038-splitting/55d0183f16d2%2C45651%2C1733716094038.1733716095554 to hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/oldWALs 2024-12-09T03:48:33,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-09T03:48:33,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 55d0183f16d2%2C45651%2C1733716094038.1733716095554, worker=55d0183f16d2,35501,1733716094084 in 238 msec 2024-12-09T03:48:33,334 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=55d0183f16d2,35501,1733716094084 2024-12-09T03:48:33,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T03:48:33,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 55d0183f16d2%2C45651%2C1733716094038.1733716095554, worker=55d0183f16d2,35501,1733716094084 in 255 msec 2024-12-09T03:48:33,340 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038-splitting dir is empty, no logs to split. 2024-12-09T03:48:33,340 INFO [PEWorker-1 {}] master.SplitWALManager(105): 55d0183f16d2,45651,1733716094038 WAL count=0, meta=false 2024-12-09T03:48:33,340 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 55d0183f16d2,45651,1733716094038 WAL splitting is done? wals=0, meta=false 2024-12-09T03:48:33,342 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 55d0183f16d2,45651,1733716094038 failed, ignore...File hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,45651,1733716094038-splitting does not exist. 2024-12-09T03:48:33,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, ASSIGN}] 2024-12-09T03:48:33,347 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, ASSIGN 2024-12-09T03:48:33,349 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T03:48:33,476 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=18] 2024-12-09T03:48:33,477 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 55d0183f16d2:45651 this server is in the failed servers list 2024-12-09T03:48:33,477 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=55d0183f16d2:45651 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 55d0183f16d2:45651 2024-12-09T03:48:33,477 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 55d0183f16d2:45651 2024-12-09T03:48:33,477 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,45651,1733716094038, seqNum=18 from cache 2024-12-09T03:48:33,499 DEBUG [55d0183f16d2:34431 {}] balancer.BalancerClusterState(204): Hosts are {55d0183f16d2=0} racks are {/default-rack=0} 2024-12-09T03:48:33,499 DEBUG [55d0183f16d2:34431 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T03:48:33,499 DEBUG [55d0183f16d2:34431 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T03:48:33,499 DEBUG [55d0183f16d2:34431 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T03:48:33,499 DEBUG [55d0183f16d2:34431 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T03:48:33,500 INFO [55d0183f16d2:34431 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T03:48:33,500 INFO [55d0183f16d2:34431 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T03:48:33,500 DEBUG [55d0183f16d2:34431 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T03:48:33,500 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=940a5def324c1e7ca6d1994726e09abd, regionState=OPENING, regionLocation=55d0183f16d2,35501,1733716094084 2024-12-09T03:48:33,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, ASSIGN because future has completed 2024-12-09T03:48:33,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,35501,1733716094084}] 2024-12-09T03:48:33,610 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-09T03:48:33,611 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-09T03:48:33,612 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-09T03:48:33,612 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-09T03:48:33,661 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:33,662 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 940a5def324c1e7ca6d1994726e09abd, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:33,662 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:33,662 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:33,662 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:33,662 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:33,664 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:33,664 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 940a5def324c1e7ca6d1994726e09abd columnFamilyName cf1 2024-12-09T03:48:33,665 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:33,672 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf1/322930b6d2c9423c9af12d5203daaf38 2024-12-09T03:48:33,673 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(327): Store=940a5def324c1e7ca6d1994726e09abd/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:33,673 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:33,674 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 940a5def324c1e7ca6d1994726e09abd columnFamilyName cf2 2024-12-09T03:48:33,674 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:33,682 DEBUG [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/cf2/7cd795225f9a4214ade3f4d3e5727404 2024-12-09T03:48:33,682 INFO [StoreOpener-940a5def324c1e7ca6d1994726e09abd-1 {}] regionserver.HStore(327): Store=940a5def324c1e7ca6d1994726e09abd/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:33,683 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:33,684 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:33,685 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:33,686 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:33,686 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:33,687 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T03:48:33,688 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:33,690 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 940a5def324c1e7ca6d1994726e09abd; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60200377, jitterRate=-0.10294447839260101}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T03:48:33,690 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:33,690 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 940a5def324c1e7ca6d1994726e09abd: Running coprocessor pre-open hook at 1733716113662Writing region info on filesystem at 1733716113662Initializing all the Stores at 1733716113663 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716113663Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716113663Cleaning up temporary data from old regions at 1733716113686 (+23 ms)Running coprocessor post-open hooks at 1733716113690 (+4 ms)Region opened successfully at 1733716113690 2024-12-09T03:48:33,692 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., pid=17, masterSystemTime=1733716113656 2024-12-09T03:48:33,694 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:33,694 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:33,695 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=940a5def324c1e7ca6d1994726e09abd, regionState=OPEN, openSeqNum=18, regionLocation=55d0183f16d2,35501,1733716094084 2024-12-09T03:48:33,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,35501,1733716094084 because future has completed 2024-12-09T03:48:33,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-09T03:48:33,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 940a5def324c1e7ca6d1994726e09abd, server=55d0183f16d2,35501,1733716094084 in 196 msec 2024-12-09T03:48:33,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-09T03:48:33,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=940a5def324c1e7ca6d1994726e09abd, ASSIGN in 358 msec 2024-12-09T03:48:33,708 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 55d0183f16d2,45651,1733716094038 after splitting done 2024-12-09T03:48:33,711 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 55d0183f16d2,45651,1733716094038, splitWal=true, meta=false in 646 msec 2024-12-09T03:48:33,986 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd., hostname=55d0183f16d2,35501,1733716094084, seqNum=18] 2024-12-09T03:48:34,006 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=408 (was 410), OpenFileDescriptor=1050 (was 1025) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=235 (was 220) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12119 (was 12441) 2024-12-09T03:48:34,006 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-12-09T03:48:34,019 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=408, OpenFileDescriptor=1050, MaxFileDescriptor=1048576, SystemLoadAverage=235, ProcessCount=11, AvailableMemoryMB=12118 2024-12-09T03:48:34,019 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-12-09T03:48:34,037 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:34,039 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:34,040 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:48:34,044 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-77213737, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/hregion-77213737, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:34,060 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-77213737/hregion-77213737.1733716114044, exclude list is [], retry=0 2024-12-09T03:48:34,063 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:34,063 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:34,064 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:34,092 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-77213737/hregion-77213737.1733716114044 2024-12-09T03:48:34,092 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:37281:37281)] 2024-12-09T03:48:34,092 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => df37bdb468c07fe341f55eb17d724fc4, NAME => 'testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35437/hbase 2024-12-09T03:48:34,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741893_1071 (size=67) 2024-12-09T03:48:34,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741893_1071 (size=67) 2024-12-09T03:48:34,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741893_1071 (size=67) 2024-12-09T03:48:34,102 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:34,104 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,105 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df37bdb468c07fe341f55eb17d724fc4 columnFamilyName a 2024-12-09T03:48:34,105 DEBUG [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:34,106 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(327): Store=df37bdb468c07fe341f55eb17d724fc4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:34,106 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,107 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df37bdb468c07fe341f55eb17d724fc4 columnFamilyName b 2024-12-09T03:48:34,107 DEBUG [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:34,108 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(327): Store=df37bdb468c07fe341f55eb17d724fc4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:34,108 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,109 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df37bdb468c07fe341f55eb17d724fc4 columnFamilyName c 2024-12-09T03:48:34,109 DEBUG [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:34,110 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(327): Store=df37bdb468c07fe341f55eb17d724fc4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:34,110 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,110 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,111 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,111 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,111 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,112 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:34,113 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,115 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:48:34,115 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened df37bdb468c07fe341f55eb17d724fc4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65096392, jitterRate=-0.02998816967010498}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:34,116 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for df37bdb468c07fe341f55eb17d724fc4: Writing region info on filesystem at 1733716114102Initializing all the Stores at 1733716114103 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716114103Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716114103Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716114103Cleaning up temporary data from old regions at 1733716114111 (+8 ms)Region opened successfully at 1733716114116 (+5 ms) 2024-12-09T03:48:34,116 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing df37bdb468c07fe341f55eb17d724fc4, disabling compactions & flushes 2024-12-09T03:48:34,116 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4. 2024-12-09T03:48:34,116 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4. 2024-12-09T03:48:34,116 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4. after waiting 0 ms 2024-12-09T03:48:34,116 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4. 2024-12-09T03:48:34,117 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4. 2024-12-09T03:48:34,117 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for df37bdb468c07fe341f55eb17d724fc4: Waiting for close lock at 1733716114116Disabling compacts and flushes for region at 1733716114116Disabling writes for close at 1733716114116Writing region close event to WAL at 1733716114117 (+1 ms)Closed at 1733716114117 2024-12-09T03:48:34,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741892_1070 (size=175) 2024-12-09T03:48:34,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741892_1070 (size=175) 2024-12-09T03:48:34,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741892_1070 (size=175) 2024-12-09T03:48:34,122 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T03:48:34,122 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-77213737:(num 1733716114044) 2024-12-09T03:48:34,122 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:34,123 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:34,137 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114124, exclude list is [], retry=0 2024-12-09T03:48:34,140 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:34,141 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:34,141 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:34,168 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114124 2024-12-09T03:48:34,168 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:36521:36521)] 2024-12-09T03:48:34,168 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => df37bdb468c07fe341f55eb17d724fc4, NAME => 'testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:34,168 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:34,169 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,169 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,170 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,171 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df37bdb468c07fe341f55eb17d724fc4 columnFamilyName a 2024-12-09T03:48:34,171 DEBUG [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:34,171 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(327): Store=df37bdb468c07fe341f55eb17d724fc4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:34,171 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,172 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df37bdb468c07fe341f55eb17d724fc4 columnFamilyName b 2024-12-09T03:48:34,172 DEBUG [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:34,173 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(327): Store=df37bdb468c07fe341f55eb17d724fc4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:34,173 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,173 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df37bdb468c07fe341f55eb17d724fc4 columnFamilyName c 2024-12-09T03:48:34,173 DEBUG [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:34,174 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(327): Store=df37bdb468c07fe341f55eb17d724fc4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:34,174 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,175 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,176 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,176 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,177 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,177 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:34,178 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,179 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened df37bdb468c07fe341f55eb17d724fc4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63473345, jitterRate=-0.05417345464229584}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:34,179 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for df37bdb468c07fe341f55eb17d724fc4: Writing region info on filesystem at 1733716114169Initializing all the Stores at 1733716114169Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716114170 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716114170Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716114170Cleaning up temporary data from old regions at 1733716114177 (+7 ms)Region opened successfully at 1733716114179 (+2 ms) 2024-12-09T03:48:34,204 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing df37bdb468c07fe341f55eb17d724fc4 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-09T03:48:34,223 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/.tmp/a/5d8262df4f8c426e91d4b6eb930a2b29 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733716114179/Put/seqid=0 2024-12-09T03:48:34,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741895_1073 (size=5958) 2024-12-09T03:48:34,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741895_1073 (size=5958) 2024-12-09T03:48:34,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741895_1073 (size=5958) 2024-12-09T03:48:34,235 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/.tmp/a/5d8262df4f8c426e91d4b6eb930a2b29 2024-12-09T03:48:34,256 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/.tmp/b/d904bec075ac490d9116ea5d3049e07d is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733716114187/Put/seqid=0 2024-12-09T03:48:34,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741896_1074 (size=5958) 2024-12-09T03:48:34,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741896_1074 (size=5958) 2024-12-09T03:48:34,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741896_1074 (size=5958) 2024-12-09T03:48:34,265 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/.tmp/b/d904bec075ac490d9116ea5d3049e07d 2024-12-09T03:48:34,286 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/.tmp/c/5ce4a853672544fd90848359ddbd0ffd is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733716114195/Put/seqid=0 2024-12-09T03:48:34,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741897_1075 (size=5958) 2024-12-09T03:48:34,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741897_1075 (size=5958) 2024-12-09T03:48:34,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741897_1075 (size=5958) 2024-12-09T03:48:34,294 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/.tmp/c/5ce4a853672544fd90848359ddbd0ffd 2024-12-09T03:48:34,302 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/.tmp/a/5d8262df4f8c426e91d4b6eb930a2b29 as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/a/5d8262df4f8c426e91d4b6eb930a2b29 2024-12-09T03:48:34,308 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/a/5d8262df4f8c426e91d4b6eb930a2b29, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T03:48:34,309 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/.tmp/b/d904bec075ac490d9116ea5d3049e07d as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/b/d904bec075ac490d9116ea5d3049e07d 2024-12-09T03:48:34,315 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/b/d904bec075ac490d9116ea5d3049e07d, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T03:48:34,316 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/.tmp/c/5ce4a853672544fd90848359ddbd0ffd as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/c/5ce4a853672544fd90848359ddbd0ffd 2024-12-09T03:48:34,322 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/c/5ce4a853672544fd90848359ddbd0ffd, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T03:48:34,323 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for df37bdb468c07fe341f55eb17d724fc4 in 119ms, sequenceid=33, compaction requested=false 2024-12-09T03:48:34,324 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for df37bdb468c07fe341f55eb17d724fc4: 2024-12-09T03:48:34,324 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing df37bdb468c07fe341f55eb17d724fc4, disabling compactions & flushes 2024-12-09T03:48:34,324 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4. 2024-12-09T03:48:34,324 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4. 2024-12-09T03:48:34,324 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4. after waiting 0 ms 2024-12-09T03:48:34,324 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4. 2024-12-09T03:48:34,325 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4. 2024-12-09T03:48:34,325 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for df37bdb468c07fe341f55eb17d724fc4: Waiting for close lock at 1733716114324Disabling compacts and flushes for region at 1733716114324Disabling writes for close at 1733716114324Writing region close event to WAL at 1733716114325 (+1 ms)Closed at 1733716114325 2024-12-09T03:48:34,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741894_1072 (size=6311) 2024-12-09T03:48:34,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741894_1072 (size=6311) 2024-12-09T03:48:34,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741894_1072 (size=6311) 2024-12-09T03:48:34,336 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/b/d904bec075ac490d9116ea5d3049e07d to hdfs://localhost:35437/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/b/d904bec075ac490d9116ea5d3049e07d 2024-12-09T03:48:34,362 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35437/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114124, size=6.2 K (6311bytes) 2024-12-09T03:48:34,362 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35437/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114124 2024-12-09T03:48:34,362 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35437/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114124 after 0ms 2024-12-09T03:48:34,391 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:34,391 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35437/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114124 took 29ms 2024-12-09T03:48:34,395 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35437/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114124 so closing down 2024-12-09T03:48:34,395 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T03:48:34,397 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733716114124.temp 2024-12-09T03:48:34,432 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/recovered.edits/0000000000000000003-wal.1733716114124.temp 2024-12-09T03:48:34,432 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T03:48:34,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741898_1076 (size=5575) 2024-12-09T03:48:34,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741898_1076 (size=5575) 2024-12-09T03:48:34,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741898_1076 (size=5575) 2024-12-09T03:48:34,443 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/recovered.edits/0000000000000000003-wal.1733716114124.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T03:48:34,445 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/recovered.edits/0000000000000000003-wal.1733716114124.temp to hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/recovered.edits/0000000000000000032 2024-12-09T03:48:34,445 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 53 ms; skipped=2; WAL=hdfs://localhost:35437/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114124, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-12-09T03:48:34,445 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35437/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114124, journal: Splitting hdfs://localhost:35437/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114124, size=6.2 K (6311bytes) at 1733716114362Finishing writing output for hdfs://localhost:35437/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114124 so closing down at 1733716114395 (+33 ms)Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/recovered.edits/0000000000000000003-wal.1733716114124.temp at 1733716114432 (+37 ms)3 split writer threads finished at 1733716114433 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/recovered.edits/0000000000000000003-wal.1733716114124.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733716114443 (+10 ms)Rename recovered edits hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/recovered.edits/0000000000000000003-wal.1733716114124.temp to hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/recovered.edits/0000000000000000032 at 1733716114445 (+2 ms)Processed 32 edits across 1 Regions in 53 ms; skipped=2; WAL=hdfs://localhost:35437/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114124, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1733716114445 2024-12-09T03:48:34,447 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35437/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114124 to hdfs://localhost:35437/hbase/oldWALs/wal.1733716114124 2024-12-09T03:48:34,448 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/recovered.edits/0000000000000000032 2024-12-09T03:48:34,448 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:34,450 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:34,463 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114450, exclude list is [], retry=0 2024-12-09T03:48:34,466 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:34,467 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:34,467 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:34,494 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114450 2024-12-09T03:48:34,495 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:37281:37281)] 2024-12-09T03:48:34,495 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => df37bdb468c07fe341f55eb17d724fc4, NAME => 'testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:34,495 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:34,495 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,495 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,497 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,498 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df37bdb468c07fe341f55eb17d724fc4 columnFamilyName a 2024-12-09T03:48:34,498 DEBUG [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:34,503 DEBUG [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/a/5d8262df4f8c426e91d4b6eb930a2b29 2024-12-09T03:48:34,503 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(327): Store=df37bdb468c07fe341f55eb17d724fc4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:34,503 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,505 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df37bdb468c07fe341f55eb17d724fc4 columnFamilyName b 2024-12-09T03:48:34,505 DEBUG [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:34,505 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(327): Store=df37bdb468c07fe341f55eb17d724fc4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:34,505 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,506 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df37bdb468c07fe341f55eb17d724fc4 columnFamilyName c 2024-12-09T03:48:34,506 DEBUG [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:34,511 DEBUG [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/c/5ce4a853672544fd90848359ddbd0ffd 2024-12-09T03:48:34,511 INFO [StoreOpener-df37bdb468c07fe341f55eb17d724fc4-1 {}] regionserver.HStore(327): Store=df37bdb468c07fe341f55eb17d724fc4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:34,511 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,512 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,513 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,514 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/recovered.edits/0000000000000000032 2024-12-09T03:48:34,542 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:34,544 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/recovered.edits/0000000000000000032 2024-12-09T03:48:34,544 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing df37bdb468c07fe341f55eb17d724fc4 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-09T03:48:34,563 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/.tmp/b/fe5f2d5c4d7946d99affa859ff926465 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733716114187/Put/seqid=0 2024-12-09T03:48:34,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741900_1078 (size=5958) 2024-12-09T03:48:34,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741900_1078 (size=5958) 2024-12-09T03:48:34,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741900_1078 (size=5958) 2024-12-09T03:48:34,570 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/.tmp/b/fe5f2d5c4d7946d99affa859ff926465 2024-12-09T03:48:34,576 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/.tmp/b/fe5f2d5c4d7946d99affa859ff926465 as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/b/fe5f2d5c4d7946d99affa859ff926465 2024-12-09T03:48:34,581 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/b/fe5f2d5c4d7946d99affa859ff926465, entries=10, sequenceid=32, filesize=5.8 K 2024-12-09T03:48:34,582 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for df37bdb468c07fe341f55eb17d724fc4 in 38ms, sequenceid=32, compaction requested=false; wal=null 2024-12-09T03:48:34,582 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/recovered.edits/0000000000000000032 2024-12-09T03:48:34,583 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,583 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,584 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:34,585 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for df37bdb468c07fe341f55eb17d724fc4 2024-12-09T03:48:34,587 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/df37bdb468c07fe341f55eb17d724fc4/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-09T03:48:34,589 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened df37bdb468c07fe341f55eb17d724fc4; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68158100, jitterRate=0.01563483476638794}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:34,589 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for df37bdb468c07fe341f55eb17d724fc4: Writing region info on filesystem at 1733716114495Initializing all the Stores at 1733716114496 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716114496Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716114496Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716114497 (+1 ms)Obtaining lock to block concurrent updates at 1733716114544 (+47 ms)Preparing flush snapshotting stores in df37bdb468c07fe341f55eb17d724fc4 at 1733716114544Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733716114545 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1733716114038.df37bdb468c07fe341f55eb17d724fc4. at 1733716114545Flushing df37bdb468c07fe341f55eb17d724fc4/b: creating writer at 1733716114545Flushing df37bdb468c07fe341f55eb17d724fc4/b: appending metadata at 1733716114562 (+17 ms)Flushing df37bdb468c07fe341f55eb17d724fc4/b: closing flushed file at 1733716114562Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62a7b6c5: reopening flushed file at 1733716114576 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for df37bdb468c07fe341f55eb17d724fc4 in 38ms, sequenceid=32, compaction requested=false; wal=null at 1733716114582 (+6 ms)Cleaning up temporary data from old regions at 1733716114583 (+1 ms)Region opened successfully at 1733716114589 (+6 ms) 2024-12-09T03:48:34,611 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=417 (was 408) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:37722 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:36234 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:48640 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:51638 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:51866 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2031904063-172.17.0.2-1733716090120:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2031904063-172.17.0.2-1733716090120:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:40188 [Waiting for operation #44] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2031904063-172.17.0.2-1733716090120:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1126 (was 1050) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=235 (was 235), ProcessCount=11 (was 11), AvailableMemoryMB=12101 (was 12118) 2024-12-09T03:48:34,611 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1126 is superior to 1024 2024-12-09T03:48:34,624 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=417, OpenFileDescriptor=1126, MaxFileDescriptor=1048576, SystemLoadAverage=235, ProcessCount=11, AvailableMemoryMB=12100 2024-12-09T03:48:34,624 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1126 is superior to 1024 2024-12-09T03:48:34,640 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:34,642 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:34,643 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:48:34,646 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-48573238, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/hregion-48573238, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:34,660 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-48573238/hregion-48573238.1733716114646, exclude list is [], retry=0 2024-12-09T03:48:34,663 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:34,663 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:34,663 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:34,701 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-48573238/hregion-48573238.1733716114646 2024-12-09T03:48:34,701 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:37281:37281)] 2024-12-09T03:48:34,702 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f1b5eaba996739a1769aa0fcab17ebe5, NAME => 'testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35437/hbase 2024-12-09T03:48:34,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741902_1080 (size=68) 2024-12-09T03:48:34,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741902_1080 (size=68) 2024-12-09T03:48:34,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741902_1080 (size=68) 2024-12-09T03:48:34,714 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:34,715 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,716 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1b5eaba996739a1769aa0fcab17ebe5 columnFamilyName a 2024-12-09T03:48:34,716 DEBUG [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:34,717 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(327): Store=f1b5eaba996739a1769aa0fcab17ebe5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:34,717 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,718 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1b5eaba996739a1769aa0fcab17ebe5 columnFamilyName b 2024-12-09T03:48:34,718 DEBUG [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:34,718 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(327): Store=f1b5eaba996739a1769aa0fcab17ebe5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:34,719 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,720 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1b5eaba996739a1769aa0fcab17ebe5 columnFamilyName c 2024-12-09T03:48:34,720 DEBUG [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:34,720 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(327): Store=f1b5eaba996739a1769aa0fcab17ebe5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:34,720 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,721 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,721 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,722 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,722 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,723 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:34,724 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,726 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:48:34,727 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f1b5eaba996739a1769aa0fcab17ebe5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59572402, jitterRate=-0.1123020350933075}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:34,728 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f1b5eaba996739a1769aa0fcab17ebe5: Writing region info on filesystem at 1733716114714Initializing all the Stores at 1733716114715 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716114715Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716114715Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716114715Cleaning up temporary data from old regions at 1733716114722 (+7 ms)Region opened successfully at 1733716114727 (+5 ms) 2024-12-09T03:48:34,728 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f1b5eaba996739a1769aa0fcab17ebe5, disabling compactions & flushes 2024-12-09T03:48:34,728 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5. 2024-12-09T03:48:34,728 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5. 2024-12-09T03:48:34,728 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5. after waiting 0 ms 2024-12-09T03:48:34,728 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5. 2024-12-09T03:48:34,728 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5. 2024-12-09T03:48:34,728 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f1b5eaba996739a1769aa0fcab17ebe5: Waiting for close lock at 1733716114728Disabling compacts and flushes for region at 1733716114728Disabling writes for close at 1733716114728Writing region close event to WAL at 1733716114728Closed at 1733716114728 2024-12-09T03:48:34,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741901_1079 (size=175) 2024-12-09T03:48:34,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741901_1079 (size=175) 2024-12-09T03:48:34,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741901_1079 (size=175) 2024-12-09T03:48:34,737 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T03:48:34,737 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-48573238:(num 1733716114646) 2024-12-09T03:48:34,737 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:34,739 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:34,753 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716114739, exclude list is [], retry=0 2024-12-09T03:48:34,756 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:34,757 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:34,757 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:34,794 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716114739 2024-12-09T03:48:34,794 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:36521:36521)] 2024-12-09T03:48:34,847 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f1b5eaba996739a1769aa0fcab17ebe5, NAME => 'testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:34,850 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,850 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:34,850 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,850 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,852 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,854 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1b5eaba996739a1769aa0fcab17ebe5 columnFamilyName a 2024-12-09T03:48:34,854 DEBUG [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:34,855 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(327): Store=f1b5eaba996739a1769aa0fcab17ebe5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:34,855 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,856 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1b5eaba996739a1769aa0fcab17ebe5 columnFamilyName b 2024-12-09T03:48:34,856 DEBUG [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:34,857 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(327): Store=f1b5eaba996739a1769aa0fcab17ebe5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:34,857 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,858 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1b5eaba996739a1769aa0fcab17ebe5 columnFamilyName c 2024-12-09T03:48:34,858 DEBUG [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:34,858 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(327): Store=f1b5eaba996739a1769aa0fcab17ebe5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:34,859 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,860 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,861 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,862 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,862 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,863 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:34,864 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,865 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f1b5eaba996739a1769aa0fcab17ebe5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71924630, jitterRate=0.07176050543785095}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:34,865 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:34,866 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f1b5eaba996739a1769aa0fcab17ebe5: Running coprocessor pre-open hook at 1733716114850Writing region info on filesystem at 1733716114850Initializing all the Stores at 1733716114851 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716114851Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716114851Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716114852 (+1 ms)Cleaning up temporary data from old regions at 1733716114862 (+10 ms)Running coprocessor post-open hooks at 1733716114865 (+3 ms)Region opened successfully at 1733716114866 (+1 ms) 2024-12-09T03:48:34,881 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f1b5eaba996739a1769aa0fcab17ebe5 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-09T03:48:34,882 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f1b5eaba996739a1769aa0fcab17ebe5/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:35,883 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f1b5eaba996739a1769aa0fcab17ebe5/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:35,904 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T03:48:36,883 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f1b5eaba996739a1769aa0fcab17ebe5/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:37,884 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f1b5eaba996739a1769aa0fcab17ebe5/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:38,885 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f1b5eaba996739a1769aa0fcab17ebe5/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:39,885 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f1b5eaba996739a1769aa0fcab17ebe5/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:40,886 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f1b5eaba996739a1769aa0fcab17ebe5/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:41,887 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f1b5eaba996739a1769aa0fcab17ebe5/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:42,318 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T03:48:42,887 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f1b5eaba996739a1769aa0fcab17ebe5/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:43,610 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-09T03:48:43,610 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-09T03:48:43,611 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-09T03:48:43,611 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-09T03:48:43,612 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-09T03:48:43,612 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-09T03:48:43,888 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for f1b5eaba996739a1769aa0fcab17ebe5/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:43,889 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f1b5eaba996739a1769aa0fcab17ebe5: 2024-12-09T03:48:43,889 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5. 2024-12-09T03:48:43,903 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f1b5eaba996739a1769aa0fcab17ebe5: 2024-12-09T03:48:43,903 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-09T03:48:43,903 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f1b5eaba996739a1769aa0fcab17ebe5, disabling compactions & flushes 2024-12-09T03:48:43,903 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5. 2024-12-09T03:48:43,903 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5. 2024-12-09T03:48:43,903 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5. after waiting 0 ms 2024-12-09T03:48:43,903 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5. 2024-12-09T03:48:43,904 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5. 2024-12-09T03:48:43,904 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5. 2024-12-09T03:48:43,904 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f1b5eaba996739a1769aa0fcab17ebe5: Waiting for close lock at 1733716123903Running coprocessor pre-close hooks at 1733716123903Disabling compacts and flushes for region at 1733716123903Disabling writes for close at 1733716123903Writing region close event to WAL at 1733716123904 (+1 ms)Running coprocessor post-close hooks at 1733716123904Closed at 1733716123904 2024-12-09T03:48:43,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741903_1081 (size=4236) 2024-12-09T03:48:43,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741903_1081 (size=4236) 2024-12-09T03:48:43,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741903_1081 (size=4236) 2024-12-09T03:48:43,923 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35437/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716114739, size=4.1 K (4236bytes) 2024-12-09T03:48:43,923 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35437/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716114739 2024-12-09T03:48:43,923 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35437/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716114739 after 0ms 2024-12-09T03:48:43,952 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:43,952 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35437/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716114739 took 29ms 2024-12-09T03:48:43,954 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35437/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716114739 so closing down 2024-12-09T03:48:43,954 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T03:48:43,956 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733716114739.temp 2024-12-09T03:48:43,982 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/recovered.edits/0000000000000000004-wal.1733716114739.temp 2024-12-09T03:48:43,983 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T03:48:43,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741904_1082 (size=3245) 2024-12-09T03:48:43,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741904_1082 (size=3245) 2024-12-09T03:48:43,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741904_1082 (size=3245) 2024-12-09T03:48:43,991 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/recovered.edits/0000000000000000004-wal.1733716114739.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-09T03:48:43,993 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/recovered.edits/0000000000000000004-wal.1733716114739.temp to hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/recovered.edits/0000000000000000026 2024-12-09T03:48:43,993 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 41 ms; skipped=3; WAL=hdfs://localhost:35437/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716114739, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-12-09T03:48:43,993 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35437/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716114739, journal: Splitting hdfs://localhost:35437/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716114739, size=4.1 K (4236bytes) at 1733716123923Finishing writing output for hdfs://localhost:35437/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716114739 so closing down at 1733716123954 (+31 ms)Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/recovered.edits/0000000000000000004-wal.1733716114739.temp at 1733716123982 (+28 ms)3 split writer threads finished at 1733716123983 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/recovered.edits/0000000000000000004-wal.1733716114739.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733716123991 (+8 ms)Rename recovered edits hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/recovered.edits/0000000000000000004-wal.1733716114739.temp to hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/recovered.edits/0000000000000000026 at 1733716123993 (+2 ms)Processed 23 edits across 1 Regions in 41 ms; skipped=3; WAL=hdfs://localhost:35437/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716114739, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1733716123993 2024-12-09T03:48:43,995 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35437/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716114739 to hdfs://localhost:35437/hbase/oldWALs/wal.1733716114739 2024-12-09T03:48:43,996 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/recovered.edits/0000000000000000026 2024-12-09T03:48:43,996 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:43,997 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:44,011 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716123998, exclude list is [], retry=0 2024-12-09T03:48:44,013 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:44,014 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:44,014 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:44,041 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716123998 2024-12-09T03:48:44,042 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:37281:37281)] 2024-12-09T03:48:44,042 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f1b5eaba996739a1769aa0fcab17ebe5, NAME => 'testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:44,043 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:44,043 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:44,043 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:44,043 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:44,045 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:44,046 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1b5eaba996739a1769aa0fcab17ebe5 columnFamilyName a 2024-12-09T03:48:44,046 DEBUG [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:44,047 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(327): Store=f1b5eaba996739a1769aa0fcab17ebe5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:44,047 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:44,048 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1b5eaba996739a1769aa0fcab17ebe5 columnFamilyName b 2024-12-09T03:48:44,048 DEBUG [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:44,049 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(327): Store=f1b5eaba996739a1769aa0fcab17ebe5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:44,049 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:44,050 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1b5eaba996739a1769aa0fcab17ebe5 columnFamilyName c 2024-12-09T03:48:44,050 DEBUG [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:44,050 INFO [StoreOpener-f1b5eaba996739a1769aa0fcab17ebe5-1 {}] regionserver.HStore(327): Store=f1b5eaba996739a1769aa0fcab17ebe5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:44,051 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:44,051 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:44,053 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:44,054 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/recovered.edits/0000000000000000026 2024-12-09T03:48:44,084 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:44,086 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/recovered.edits/0000000000000000026 2024-12-09T03:48:44,086 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f1b5eaba996739a1769aa0fcab17ebe5 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-09T03:48:44,104 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/.tmp/a/660795d2d0a84a489c997626da7a6daf is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733716123894/Put/seqid=0 2024-12-09T03:48:44,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741906_1084 (size=5523) 2024-12-09T03:48:44,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741906_1084 (size=5523) 2024-12-09T03:48:44,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741906_1084 (size=5523) 2024-12-09T03:48:44,112 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/.tmp/a/660795d2d0a84a489c997626da7a6daf 2024-12-09T03:48:44,139 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/.tmp/b/473e954b8fc941d08f2995b02d180a40 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733716123889/Put/seqid=0 2024-12-09T03:48:44,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741907_1085 (size=5524) 2024-12-09T03:48:44,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741907_1085 (size=5524) 2024-12-09T03:48:44,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741907_1085 (size=5524) 2024-12-09T03:48:44,147 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/.tmp/b/473e954b8fc941d08f2995b02d180a40 2024-12-09T03:48:44,167 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/.tmp/c/148c805ff2fd40f9bc9c47067e7a2f8a is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733716123892/Put/seqid=0 2024-12-09T03:48:44,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741908_1086 (size=5457) 2024-12-09T03:48:44,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741908_1086 (size=5457) 2024-12-09T03:48:44,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741908_1086 (size=5457) 2024-12-09T03:48:44,175 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/.tmp/c/148c805ff2fd40f9bc9c47067e7a2f8a 2024-12-09T03:48:44,180 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/.tmp/a/660795d2d0a84a489c997626da7a6daf as hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/a/660795d2d0a84a489c997626da7a6daf 2024-12-09T03:48:44,185 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/a/660795d2d0a84a489c997626da7a6daf, entries=7, sequenceid=26, filesize=5.4 K 2024-12-09T03:48:44,186 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/.tmp/b/473e954b8fc941d08f2995b02d180a40 as hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/b/473e954b8fc941d08f2995b02d180a40 2024-12-09T03:48:44,190 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/b/473e954b8fc941d08f2995b02d180a40, entries=7, sequenceid=26, filesize=5.4 K 2024-12-09T03:48:44,191 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/.tmp/c/148c805ff2fd40f9bc9c47067e7a2f8a as hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/c/148c805ff2fd40f9bc9c47067e7a2f8a 2024-12-09T03:48:44,195 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/c/148c805ff2fd40f9bc9c47067e7a2f8a, entries=6, sequenceid=26, filesize=5.3 K 2024-12-09T03:48:44,196 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for f1b5eaba996739a1769aa0fcab17ebe5 in 110ms, sequenceid=26, compaction requested=false; wal=null 2024-12-09T03:48:44,196 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/recovered.edits/0000000000000000026 2024-12-09T03:48:44,197 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:44,197 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:44,198 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:44,200 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:44,201 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testReplayEditsAfterAbortingFlush/f1b5eaba996739a1769aa0fcab17ebe5/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-09T03:48:44,202 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f1b5eaba996739a1769aa0fcab17ebe5; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59065422, jitterRate=-0.11985662579536438}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:44,202 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f1b5eaba996739a1769aa0fcab17ebe5 2024-12-09T03:48:44,203 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f1b5eaba996739a1769aa0fcab17ebe5: Running coprocessor pre-open hook at 1733716124043Writing region info on filesystem at 1733716124043Initializing all the Stores at 1733716124044 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716124044Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716124044Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716124044Obtaining lock to block concurrent updates at 1733716124086 (+42 ms)Preparing flush snapshotting stores in f1b5eaba996739a1769aa0fcab17ebe5 at 1733716124086Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733716124086Flushing stores of testReplayEditsAfterAbortingFlush,,1733716114641.f1b5eaba996739a1769aa0fcab17ebe5. at 1733716124086Flushing f1b5eaba996739a1769aa0fcab17ebe5/a: creating writer at 1733716124086Flushing f1b5eaba996739a1769aa0fcab17ebe5/a: appending metadata at 1733716124104 (+18 ms)Flushing f1b5eaba996739a1769aa0fcab17ebe5/a: closing flushed file at 1733716124104Flushing f1b5eaba996739a1769aa0fcab17ebe5/b: creating writer at 1733716124118 (+14 ms)Flushing f1b5eaba996739a1769aa0fcab17ebe5/b: appending metadata at 1733716124139 (+21 ms)Flushing f1b5eaba996739a1769aa0fcab17ebe5/b: closing flushed file at 1733716124139Flushing f1b5eaba996739a1769aa0fcab17ebe5/c: creating writer at 1733716124152 (+13 ms)Flushing f1b5eaba996739a1769aa0fcab17ebe5/c: appending metadata at 1733716124167 (+15 ms)Flushing f1b5eaba996739a1769aa0fcab17ebe5/c: closing flushed file at 1733716124167Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e33c8e: reopening flushed file at 1733716124180 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35c0bc36: reopening flushed file at 1733716124185 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@326a40c8: reopening flushed file at 1733716124190 (+5 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for f1b5eaba996739a1769aa0fcab17ebe5 in 110ms, sequenceid=26, compaction requested=false; wal=null at 1733716124196 (+6 ms)Cleaning up temporary data from old regions at 1733716124197 (+1 ms)Running coprocessor post-open hooks at 1733716124202 (+5 ms)Region opened successfully at 1733716124203 (+1 ms) 2024-12-09T03:48:44,224 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=413 (was 417), OpenFileDescriptor=1182 (was 1126) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 235), ProcessCount=11 (was 11), AvailableMemoryMB=12085 (was 12100) 2024-12-09T03:48:44,224 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-09T03:48:44,238 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=413, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=12084 2024-12-09T03:48:44,238 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-09T03:48:44,253 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:44,255 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:44,255 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:48:44,258 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-80268634, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/hregion-80268634, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:44,271 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-80268634/hregion-80268634.1733716124258, exclude list is [], retry=0 2024-12-09T03:48:44,273 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:44,274 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:44,274 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:44,314 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-80268634/hregion-80268634.1733716124258 2024-12-09T03:48:44,314 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:37281:37281)] 2024-12-09T03:48:44,315 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 82eb8427216c2a7d07cc814c7ffd8a4f, NAME => 'testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35437/hbase 2024-12-09T03:48:44,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741910_1088 (size=61) 2024-12-09T03:48:44,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741910_1088 (size=61) 2024-12-09T03:48:44,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741910_1088 (size=61) 2024-12-09T03:48:44,324 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:44,325 INFO [StoreOpener-82eb8427216c2a7d07cc814c7ffd8a4f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,327 INFO [StoreOpener-82eb8427216c2a7d07cc814c7ffd8a4f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82eb8427216c2a7d07cc814c7ffd8a4f columnFamilyName a 2024-12-09T03:48:44,327 DEBUG [StoreOpener-82eb8427216c2a7d07cc814c7ffd8a4f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:44,327 INFO [StoreOpener-82eb8427216c2a7d07cc814c7ffd8a4f-1 {}] regionserver.HStore(327): Store=82eb8427216c2a7d07cc814c7ffd8a4f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:44,327 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,328 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,328 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,329 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,329 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,330 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,333 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:48:44,333 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 82eb8427216c2a7d07cc814c7ffd8a4f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70199872, jitterRate=0.046059608459472656}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T03:48:44,334 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 82eb8427216c2a7d07cc814c7ffd8a4f: Writing region info on filesystem at 1733716124324Initializing all the Stores at 1733716124325 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716124325Cleaning up temporary data from old regions at 1733716124329 (+4 ms)Region opened successfully at 1733716124334 (+5 ms) 2024-12-09T03:48:44,334 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 82eb8427216c2a7d07cc814c7ffd8a4f, disabling compactions & flushes 2024-12-09T03:48:44,334 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f. 2024-12-09T03:48:44,334 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f. 2024-12-09T03:48:44,334 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f. after waiting 0 ms 2024-12-09T03:48:44,335 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f. 2024-12-09T03:48:44,335 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f. 2024-12-09T03:48:44,335 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 82eb8427216c2a7d07cc814c7ffd8a4f: Waiting for close lock at 1733716124334Disabling compacts and flushes for region at 1733716124334Disabling writes for close at 1733716124335 (+1 ms)Writing region close event to WAL at 1733716124335Closed at 1733716124335 2024-12-09T03:48:44,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741909_1087 (size=175) 2024-12-09T03:48:44,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741909_1087 (size=175) 2024-12-09T03:48:44,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741909_1087 (size=175) 2024-12-09T03:48:44,341 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T03:48:44,341 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-80268634:(num 1733716124258) 2024-12-09T03:48:44,341 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:44,343 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:44,357 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124343, exclude list is [], retry=0 2024-12-09T03:48:44,360 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:44,360 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:44,361 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:44,388 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124343 2024-12-09T03:48:44,388 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:36521:36521)] 2024-12-09T03:48:44,388 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 82eb8427216c2a7d07cc814c7ffd8a4f, NAME => 'testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:44,388 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:44,389 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,389 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,390 INFO [StoreOpener-82eb8427216c2a7d07cc814c7ffd8a4f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,391 INFO [StoreOpener-82eb8427216c2a7d07cc814c7ffd8a4f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82eb8427216c2a7d07cc814c7ffd8a4f columnFamilyName a 2024-12-09T03:48:44,391 DEBUG [StoreOpener-82eb8427216c2a7d07cc814c7ffd8a4f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:44,391 INFO [StoreOpener-82eb8427216c2a7d07cc814c7ffd8a4f-1 {}] regionserver.HStore(327): Store=82eb8427216c2a7d07cc814c7ffd8a4f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:44,392 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,393 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,394 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,394 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,394 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,396 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,396 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 82eb8427216c2a7d07cc814c7ffd8a4f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64512709, jitterRate=-0.03868572413921356}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T03:48:44,397 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 82eb8427216c2a7d07cc814c7ffd8a4f: Writing region info on filesystem at 1733716124389Initializing all the Stores at 1733716124390 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716124390Cleaning up temporary data from old regions at 1733716124394 (+4 ms)Region opened successfully at 1733716124397 (+3 ms) 2024-12-09T03:48:44,406 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 82eb8427216c2a7d07cc814c7ffd8a4f, disabling compactions & flushes 2024-12-09T03:48:44,406 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f. 2024-12-09T03:48:44,406 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f. 2024-12-09T03:48:44,406 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f. after waiting 0 ms 2024-12-09T03:48:44,406 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f. 2024-12-09T03:48:44,406 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f. 2024-12-09T03:48:44,406 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f. 2024-12-09T03:48:44,406 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 82eb8427216c2a7d07cc814c7ffd8a4f: Waiting for close lock at 1733716124406Disabling compacts and flushes for region at 1733716124406Disabling writes for close at 1733716124406Writing region close event to WAL at 1733716124406Closed at 1733716124406 2024-12-09T03:48:44,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741911_1089 (size=1795) 2024-12-09T03:48:44,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741911_1089 (size=1795) 2024-12-09T03:48:44,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741911_1089 (size=1795) 2024-12-09T03:48:44,426 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35437/hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124343, size=1.8 K (1795bytes) 2024-12-09T03:48:44,426 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35437/hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124343 2024-12-09T03:48:44,426 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35437/hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124343 after 0ms 2024-12-09T03:48:44,454 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:44,454 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35437/hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124343 took 28ms 2024-12-09T03:48:44,456 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35437/hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124343 so closing down 2024-12-09T03:48:44,456 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T03:48:44,458 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733716124343.temp 2024-12-09T03:48:44,484 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/recovered.edits/0000000000000000003-wal.1733716124343.temp 2024-12-09T03:48:44,485 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T03:48:44,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741912_1090 (size=1795) 2024-12-09T03:48:44,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741912_1090 (size=1795) 2024-12-09T03:48:44,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741912_1090 (size=1795) 2024-12-09T03:48:44,492 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/recovered.edits/0000000000000000003-wal.1733716124343.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-09T03:48:44,493 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/recovered.edits/0000000000000000003-wal.1733716124343.temp to hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/recovered.edits/0000000000000000012 2024-12-09T03:48:44,493 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124343, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-12-09T03:48:44,494 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35437/hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124343, journal: Splitting hdfs://localhost:35437/hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124343, size=1.8 K (1795bytes) at 1733716124426Finishing writing output for hdfs://localhost:35437/hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124343 so closing down at 1733716124456 (+30 ms)Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/recovered.edits/0000000000000000003-wal.1733716124343.temp at 1733716124484 (+28 ms)3 split writer threads finished at 1733716124485 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/recovered.edits/0000000000000000003-wal.1733716124343.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733716124492 (+7 ms)Rename recovered edits hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/recovered.edits/0000000000000000003-wal.1733716124343.temp to hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/recovered.edits/0000000000000000012 at 1733716124493 (+1 ms)Processed 10 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124343, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1733716124493 2024-12-09T03:48:44,495 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35437/hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124343 to hdfs://localhost:35437/hbase/oldWALs/wal.1733716124343 2024-12-09T03:48:44,496 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/recovered.edits/0000000000000000012 2024-12-09T03:48:44,526 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:44,825 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:44,827 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:44,841 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124828, exclude list is [], retry=0 2024-12-09T03:48:44,843 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:44,844 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:44,844 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:44,871 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124828 2024-12-09T03:48:44,871 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:36521:36521)] 2024-12-09T03:48:44,871 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 82eb8427216c2a7d07cc814c7ffd8a4f, NAME => 'testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:44,871 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:44,871 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,871 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,875 INFO [StoreOpener-82eb8427216c2a7d07cc814c7ffd8a4f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,876 INFO [StoreOpener-82eb8427216c2a7d07cc814c7ffd8a4f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82eb8427216c2a7d07cc814c7ffd8a4f columnFamilyName a 2024-12-09T03:48:44,876 DEBUG [StoreOpener-82eb8427216c2a7d07cc814c7ffd8a4f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:44,877 INFO [StoreOpener-82eb8427216c2a7d07cc814c7ffd8a4f-1 {}] regionserver.HStore(327): Store=82eb8427216c2a7d07cc814c7ffd8a4f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:44,877 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,878 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,879 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,880 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/recovered.edits/0000000000000000012 2024-12-09T03:48:44,907 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:44,908 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/recovered.edits/0000000000000000012 2024-12-09T03:48:44,908 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 82eb8427216c2a7d07cc814c7ffd8a4f 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-09T03:48:44,927 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/.tmp/a/f06d81a186e5423992a197bfd7b801cf is 79, key is testDatalossWhenInputError/a:x0/1733716124397/Put/seqid=0 2024-12-09T03:48:44,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741914_1092 (size=5808) 2024-12-09T03:48:44,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741914_1092 (size=5808) 2024-12-09T03:48:44,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741914_1092 (size=5808) 2024-12-09T03:48:44,935 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/.tmp/a/f06d81a186e5423992a197bfd7b801cf 2024-12-09T03:48:44,944 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/.tmp/a/f06d81a186e5423992a197bfd7b801cf as hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/a/f06d81a186e5423992a197bfd7b801cf 2024-12-09T03:48:44,952 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/a/f06d81a186e5423992a197bfd7b801cf, entries=10, sequenceid=12, filesize=5.7 K 2024-12-09T03:48:44,952 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 82eb8427216c2a7d07cc814c7ffd8a4f in 44ms, sequenceid=12, compaction requested=false; wal=null 2024-12-09T03:48:44,953 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/recovered.edits/0000000000000000012 2024-12-09T03:48:44,954 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,954 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,957 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,959 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-09T03:48:44,960 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 82eb8427216c2a7d07cc814c7ffd8a4f; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71863405, jitterRate=0.07084818184375763}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T03:48:44,961 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 82eb8427216c2a7d07cc814c7ffd8a4f: Writing region info on filesystem at 1733716124871Initializing all the Stores at 1733716124874 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716124874Obtaining lock to block concurrent updates at 1733716124908 (+34 ms)Preparing flush snapshotting stores in 82eb8427216c2a7d07cc814c7ffd8a4f at 1733716124908Finished memstore snapshotting testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733716124908Flushing stores of testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f. at 1733716124908Flushing 82eb8427216c2a7d07cc814c7ffd8a4f/a: creating writer at 1733716124908Flushing 82eb8427216c2a7d07cc814c7ffd8a4f/a: appending metadata at 1733716124926 (+18 ms)Flushing 82eb8427216c2a7d07cc814c7ffd8a4f/a: closing flushed file at 1733716124926Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3390a5a7: reopening flushed file at 1733716124943 (+17 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 82eb8427216c2a7d07cc814c7ffd8a4f in 44ms, sequenceid=12, compaction requested=false; wal=null at 1733716124952 (+9 ms)Cleaning up temporary data from old regions at 1733716124954 (+2 ms)Region opened successfully at 1733716124961 (+7 ms) 2024-12-09T03:48:44,964 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 82eb8427216c2a7d07cc814c7ffd8a4f, NAME => 'testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:44,964 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733716124254.82eb8427216c2a7d07cc814c7ffd8a4f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:44,964 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,964 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,966 INFO [StoreOpener-82eb8427216c2a7d07cc814c7ffd8a4f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,967 INFO [StoreOpener-82eb8427216c2a7d07cc814c7ffd8a4f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82eb8427216c2a7d07cc814c7ffd8a4f columnFamilyName a 2024-12-09T03:48:44,967 DEBUG [StoreOpener-82eb8427216c2a7d07cc814c7ffd8a4f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:44,972 DEBUG [StoreOpener-82eb8427216c2a7d07cc814c7ffd8a4f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/a/f06d81a186e5423992a197bfd7b801cf 2024-12-09T03:48:44,972 INFO [StoreOpener-82eb8427216c2a7d07cc814c7ffd8a4f-1 {}] regionserver.HStore(327): Store=82eb8427216c2a7d07cc814c7ffd8a4f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:44,972 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,973 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,974 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,975 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,975 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,977 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 82eb8427216c2a7d07cc814c7ffd8a4f 2024-12-09T03:48:44,979 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testDatalossWhenInputError/82eb8427216c2a7d07cc814c7ffd8a4f/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-09T03:48:44,980 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 82eb8427216c2a7d07cc814c7ffd8a4f; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61810039, jitterRate=-0.07895864546298981}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T03:48:44,980 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 82eb8427216c2a7d07cc814c7ffd8a4f: Writing region info on filesystem at 1733716124965Initializing all the Stores at 1733716124965Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716124965Cleaning up temporary data from old regions at 1733716124975 (+10 ms)Region opened successfully at 1733716124980 (+5 ms) 2024-12-09T03:48:44,998 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=423 (was 413) Potentially hanging thread: PacketResponder: BP-2031904063-172.17.0.2-1733716090120:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2031904063-172.17.0.2-1733716090120:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:60846 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:52188 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:52244 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2031904063-172.17.0.2-1733716090120:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:60780 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:37730 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=12067 (was 12084) 2024-12-09T03:48:44,998 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-09T03:48:45,011 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=423, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=12065 2024-12-09T03:48:45,011 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-09T03:48:45,026 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:45,028 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:45,029 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:48:45,031 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-89701310, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/hregion-89701310, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:45,045 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-89701310/hregion-89701310.1733716125032, exclude list is [], retry=0 2024-12-09T03:48:45,047 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:45,047 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:45,047 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:45,075 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-89701310/hregion-89701310.1733716125032 2024-12-09T03:48:45,076 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:38745:38745)] 2024-12-09T03:48:45,076 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8105ced0dcc968d57f284af6c7939248, NAME => 'testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35437/hbase 2024-12-09T03:48:45,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741916_1094 (size=63) 2024-12-09T03:48:45,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741916_1094 (size=63) 2024-12-09T03:48:45,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741916_1094 (size=63) 2024-12-09T03:48:45,085 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:45,086 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,087 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8105ced0dcc968d57f284af6c7939248 columnFamilyName a 2024-12-09T03:48:45,087 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:45,088 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(327): Store=8105ced0dcc968d57f284af6c7939248/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:45,088 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,089 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8105ced0dcc968d57f284af6c7939248 columnFamilyName b 2024-12-09T03:48:45,089 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:45,090 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(327): Store=8105ced0dcc968d57f284af6c7939248/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:45,090 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,091 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8105ced0dcc968d57f284af6c7939248 columnFamilyName c 2024-12-09T03:48:45,091 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:45,091 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(327): Store=8105ced0dcc968d57f284af6c7939248/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:45,091 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,092 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,092 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,093 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,093 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,094 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:45,095 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,098 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:48:45,098 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8105ced0dcc968d57f284af6c7939248; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58791420, jitterRate=-0.12393957376480103}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:45,099 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8105ced0dcc968d57f284af6c7939248: Writing region info on filesystem at 1733716125085Initializing all the Stores at 1733716125086 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716125086Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716125086Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716125086Cleaning up temporary data from old regions at 1733716125093 (+7 ms)Region opened successfully at 1733716125099 (+6 ms) 2024-12-09T03:48:45,099 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8105ced0dcc968d57f284af6c7939248, disabling compactions & flushes 2024-12-09T03:48:45,099 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248. 2024-12-09T03:48:45,099 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248. 2024-12-09T03:48:45,099 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248. after waiting 0 ms 2024-12-09T03:48:45,099 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248. 2024-12-09T03:48:45,100 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248. 2024-12-09T03:48:45,100 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8105ced0dcc968d57f284af6c7939248: Waiting for close lock at 1733716125099Disabling compacts and flushes for region at 1733716125099Disabling writes for close at 1733716125099Writing region close event to WAL at 1733716125099Closed at 1733716125100 (+1 ms) 2024-12-09T03:48:45,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741915_1093 (size=175) 2024-12-09T03:48:45,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741915_1093 (size=175) 2024-12-09T03:48:45,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741915_1093 (size=175) 2024-12-09T03:48:45,108 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T03:48:45,109 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-89701310:(num 1733716125032) 2024-12-09T03:48:45,109 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:45,111 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:45,158 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112, exclude list is [], retry=0 2024-12-09T03:48:45,161 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:45,161 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:45,162 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:45,202 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112 2024-12-09T03:48:45,202 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:38745:38745)] 2024-12-09T03:48:45,202 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8105ced0dcc968d57f284af6c7939248, NAME => 'testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:45,202 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:45,202 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,202 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,204 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,205 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8105ced0dcc968d57f284af6c7939248 columnFamilyName a 2024-12-09T03:48:45,205 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:45,206 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(327): Store=8105ced0dcc968d57f284af6c7939248/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:45,206 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,207 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8105ced0dcc968d57f284af6c7939248 columnFamilyName b 2024-12-09T03:48:45,207 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:45,207 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(327): Store=8105ced0dcc968d57f284af6c7939248/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:45,207 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,208 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8105ced0dcc968d57f284af6c7939248 columnFamilyName c 2024-12-09T03:48:45,208 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:45,209 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(327): Store=8105ced0dcc968d57f284af6c7939248/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:45,209 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,209 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,211 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,212 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,212 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,212 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:45,216 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:45,217 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8105ced0dcc968d57f284af6c7939248; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65043532, jitterRate=-0.030775845050811768}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:45,217 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8105ced0dcc968d57f284af6c7939248: Writing region info on filesystem at 1733716125203Initializing all the Stores at 1733716125203Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716125204 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716125204Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716125204Cleaning up temporary data from old regions at 1733716125212 (+8 ms)Region opened successfully at 1733716125217 (+5 ms) 2024-12-09T03:48:45,221 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733716125221/Put/seqid=0 2024-12-09T03:48:45,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741918_1096 (size=4875) 2024-12-09T03:48:45,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741918_1096 (size=4875) 2024-12-09T03:48:45,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741918_1096 (size=4875) 2024-12-09T03:48:45,232 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733716125231/Put/seqid=0 2024-12-09T03:48:45,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741919_1097 (size=4875) 2024-12-09T03:48:45,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741919_1097 (size=4875) 2024-12-09T03:48:45,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741919_1097 (size=4875) 2024-12-09T03:48:45,247 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733716125246/Put/seqid=0 2024-12-09T03:48:45,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741920_1098 (size=4875) 2024-12-09T03:48:45,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741920_1098 (size=4875) 2024-12-09T03:48:45,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741920_1098 (size=4875) 2024-12-09T03:48:45,257 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35437/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 8105ced0dcc968d57f284af6c7939248/a 2024-12-09T03:48:45,261 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-09T03:48:45,261 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T03:48:45,261 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35437/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 8105ced0dcc968d57f284af6c7939248/a 2024-12-09T03:48:45,265 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-09T03:48:45,265 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T03:48:45,265 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35437/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 8105ced0dcc968d57f284af6c7939248/a 2024-12-09T03:48:45,269 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-09T03:48:45,269 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T03:48:45,269 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8105ced0dcc968d57f284af6c7939248 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-09T03:48:45,285 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/.tmp/a/1bb04daf1514486d89e40cf9017ac7dd is 55, key is testCompactedBulkLoadedFiles/a:a/1733716125218/Put/seqid=0 2024-12-09T03:48:45,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741921_1099 (size=5107) 2024-12-09T03:48:45,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741921_1099 (size=5107) 2024-12-09T03:48:45,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741921_1099 (size=5107) 2024-12-09T03:48:45,292 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/.tmp/a/1bb04daf1514486d89e40cf9017ac7dd 2024-12-09T03:48:45,297 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/.tmp/a/1bb04daf1514486d89e40cf9017ac7dd as hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/1bb04daf1514486d89e40cf9017ac7dd 2024-12-09T03:48:45,302 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/1bb04daf1514486d89e40cf9017ac7dd, entries=1, sequenceid=4, filesize=5.0 K 2024-12-09T03:48:45,303 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 8105ced0dcc968d57f284af6c7939248 in 34ms, sequenceid=4, compaction requested=false 2024-12-09T03:48:45,303 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8105ced0dcc968d57f284af6c7939248: 2024-12-09T03:48:45,304 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/87f5a3d7fcdb4b5b8f92627164030a6a_SeqId_4_ 2024-12-09T03:48:45,306 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/5e7a90f29abe4ff4895241e25fcecfe8_SeqId_4_ 2024-12-09T03:48:45,307 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/a5efbbccea8a4719b6109572184c9d50_SeqId_4_ 2024-12-09T03:48:45,307 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35437/hbase/testCompactedBulkLoadedFiles/hfile0 into 8105ced0dcc968d57f284af6c7939248/a as hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/87f5a3d7fcdb4b5b8f92627164030a6a_SeqId_4_ - updating store file list. 2024-12-09T03:48:45,312 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 87f5a3d7fcdb4b5b8f92627164030a6a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T03:48:45,312 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/87f5a3d7fcdb4b5b8f92627164030a6a_SeqId_4_ into 8105ced0dcc968d57f284af6c7939248/a 2024-12-09T03:48:45,312 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35437/hbase/testCompactedBulkLoadedFiles/hfile0 into 8105ced0dcc968d57f284af6c7939248/a (new location: hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/87f5a3d7fcdb4b5b8f92627164030a6a_SeqId_4_) 2024-12-09T03:48:45,313 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35437/hbase/testCompactedBulkLoadedFiles/hfile1 into 8105ced0dcc968d57f284af6c7939248/a as hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/5e7a90f29abe4ff4895241e25fcecfe8_SeqId_4_ - updating store file list. 2024-12-09T03:48:45,317 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5e7a90f29abe4ff4895241e25fcecfe8_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T03:48:45,317 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/5e7a90f29abe4ff4895241e25fcecfe8_SeqId_4_ into 8105ced0dcc968d57f284af6c7939248/a 2024-12-09T03:48:45,317 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35437/hbase/testCompactedBulkLoadedFiles/hfile1 into 8105ced0dcc968d57f284af6c7939248/a (new location: hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/5e7a90f29abe4ff4895241e25fcecfe8_SeqId_4_) 2024-12-09T03:48:45,319 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35437/hbase/testCompactedBulkLoadedFiles/hfile2 into 8105ced0dcc968d57f284af6c7939248/a as hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/a5efbbccea8a4719b6109572184c9d50_SeqId_4_ - updating store file list. 2024-12-09T03:48:45,324 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for a5efbbccea8a4719b6109572184c9d50_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T03:48:45,324 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/a5efbbccea8a4719b6109572184c9d50_SeqId_4_ into 8105ced0dcc968d57f284af6c7939248/a 2024-12-09T03:48:45,324 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35437/hbase/testCompactedBulkLoadedFiles/hfile2 into 8105ced0dcc968d57f284af6c7939248/a (new location: hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/a5efbbccea8a4719b6109572184c9d50_SeqId_4_) 2024-12-09T03:48:45,333 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T03:48:45,333 DEBUG [Time-limited test {}] regionserver.HStore(1541): 8105ced0dcc968d57f284af6c7939248/a is initiating major compaction (all files) 2024-12-09T03:48:45,333 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 8105ced0dcc968d57f284af6c7939248/a in testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248. 2024-12-09T03:48:45,334 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/1bb04daf1514486d89e40cf9017ac7dd, hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/87f5a3d7fcdb4b5b8f92627164030a6a_SeqId_4_, hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/5e7a90f29abe4ff4895241e25fcecfe8_SeqId_4_, hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/a5efbbccea8a4719b6109572184c9d50_SeqId_4_] into tmpdir=hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/.tmp, totalSize=19.3 K 2024-12-09T03:48:45,334 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1bb04daf1514486d89e40cf9017ac7dd, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733716125218 2024-12-09T03:48:45,335 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 87f5a3d7fcdb4b5b8f92627164030a6a_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T03:48:45,335 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5e7a90f29abe4ff4895241e25fcecfe8_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T03:48:45,336 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a5efbbccea8a4719b6109572184c9d50_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T03:48:45,348 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/.tmp/a/945e34006ac64a12b4daee40811db5d0 is 55, key is testCompactedBulkLoadedFiles/a:a/1733716125218/Put/seqid=0 2024-12-09T03:48:45,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741922_1100 (size=6154) 2024-12-09T03:48:45,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741922_1100 (size=6154) 2024-12-09T03:48:45,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741922_1100 (size=6154) 2024-12-09T03:48:45,361 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/.tmp/a/945e34006ac64a12b4daee40811db5d0 as hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/945e34006ac64a12b4daee40811db5d0 2024-12-09T03:48:45,367 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 8105ced0dcc968d57f284af6c7939248/a of 8105ced0dcc968d57f284af6c7939248 into 945e34006ac64a12b4daee40811db5d0(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T03:48:45,367 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 8105ced0dcc968d57f284af6c7939248: 2024-12-09T03:48:45,367 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-09T03:48:45,367 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-09T03:48:45,399 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35437/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112, size=0 (0bytes) 2024-12-09T03:48:45,399 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35437/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112 might be still open, length is 0 2024-12-09T03:48:45,399 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35437/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112 2024-12-09T03:48:45,400 WARN [IPC Server handler 4 on default port 35437 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-09T03:48:45,401 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35437/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112 after 2ms 2024-12-09T03:48:48,144 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:60882 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:45815:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60882 dst: /127.0.0.1:45815 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45815 remote=/127.0.0.1:60882]. Total timeout mills is 60000, 57224 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:48,145 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:52284 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:39115:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52284 dst: /127.0.0.1:39115 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:48,145 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:37758 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36217:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37758 dst: /127.0.0.1:36217 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:48,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741917_1101 (size=1794) 2024-12-09T03:48:48,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741917_1101 (size=1794) 2024-12-09T03:48:48,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741917_1101 (size=1794) 2024-12-09T03:48:49,401 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35437/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112 after 4002ms 2024-12-09T03:48:49,424 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T03:48:49,429 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:49,429 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35437/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112 took 4030ms 2024-12-09T03:48:49,431 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35437/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112; continuing. 2024-12-09T03:48:49,431 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35437/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112 so closing down 2024-12-09T03:48:49,432 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T03:48:49,433 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733716125112.temp 2024-12-09T03:48:49,469 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/recovered.edits/0000000000000000003-wal.1733716125112.temp 2024-12-09T03:48:49,470 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T03:48:49,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741923_1102 (size=800) 2024-12-09T03:48:49,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741923_1102 (size=800) 2024-12-09T03:48:49,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741923_1102 (size=800) 2024-12-09T03:48:49,477 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/recovered.edits/0000000000000000003-wal.1733716125112.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T03:48:49,478 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/recovered.edits/0000000000000000003-wal.1733716125112.temp to hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/recovered.edits/0000000000000000008 2024-12-09T03:48:49,478 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 48 ms; skipped=3; WAL=hdfs://localhost:35437/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T03:48:49,478 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35437/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112, journal: Splitting hdfs://localhost:35437/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112, size=0 (0bytes) at 1733716125399Finishing writing output for hdfs://localhost:35437/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112 so closing down at 1733716129431 (+4032 ms)Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/recovered.edits/0000000000000000003-wal.1733716125112.temp at 1733716129469 (+38 ms)3 split writer threads finished at 1733716129470 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/recovered.edits/0000000000000000003-wal.1733716125112.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733716129477 (+7 ms)Rename recovered edits hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/recovered.edits/0000000000000000003-wal.1733716125112.temp to hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/recovered.edits/0000000000000000008 at 1733716129478 (+1 ms)Processed 5 edits across 1 Regions in 48 ms; skipped=3; WAL=hdfs://localhost:35437/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112, size=0, length=0, corrupted=false, cancelled=false at 1733716129478 2024-12-09T03:48:49,480 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35437/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112 to hdfs://localhost:35437/hbase/oldWALs/wal.1733716125112 2024-12-09T03:48:49,481 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/recovered.edits/0000000000000000008 2024-12-09T03:48:49,481 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:49,482 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:49,495 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716129483, exclude list is [], retry=0 2024-12-09T03:48:49,497 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:49,497 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:49,497 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:49,523 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716129483 2024-12-09T03:48:49,524 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:36521:36521)] 2024-12-09T03:48:49,524 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8105ced0dcc968d57f284af6c7939248, NAME => 'testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:49,524 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:49,524 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:49,524 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:49,525 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:49,526 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8105ced0dcc968d57f284af6c7939248 columnFamilyName a 2024-12-09T03:48:49,526 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:49,532 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/1bb04daf1514486d89e40cf9017ac7dd 2024-12-09T03:48:49,536 DEBUG [StoreFileOpener-8105ced0dcc968d57f284af6c7939248-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5e7a90f29abe4ff4895241e25fcecfe8_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T03:48:49,536 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/5e7a90f29abe4ff4895241e25fcecfe8_SeqId_4_ 2024-12-09T03:48:49,540 DEBUG [StoreFileOpener-8105ced0dcc968d57f284af6c7939248-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 87f5a3d7fcdb4b5b8f92627164030a6a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T03:48:49,540 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/87f5a3d7fcdb4b5b8f92627164030a6a_SeqId_4_ 2024-12-09T03:48:49,543 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/945e34006ac64a12b4daee40811db5d0 2024-12-09T03:48:49,547 DEBUG [StoreFileOpener-8105ced0dcc968d57f284af6c7939248-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for a5efbbccea8a4719b6109572184c9d50_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T03:48:49,547 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/a5efbbccea8a4719b6109572184c9d50_SeqId_4_ 2024-12-09T03:48:49,547 WARN [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/1bb04daf1514486d89e40cf9017ac7dd from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7f7960e3 2024-12-09T03:48:49,547 WARN [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/5e7a90f29abe4ff4895241e25fcecfe8_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7f7960e3 2024-12-09T03:48:49,547 WARN [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/87f5a3d7fcdb4b5b8f92627164030a6a_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7f7960e3 2024-12-09T03:48:49,547 WARN [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/a5efbbccea8a4719b6109572184c9d50_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7f7960e3 2024-12-09T03:48:49,547 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/1bb04daf1514486d89e40cf9017ac7dd, hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/5e7a90f29abe4ff4895241e25fcecfe8_SeqId_4_, hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/87f5a3d7fcdb4b5b8f92627164030a6a_SeqId_4_, hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/a5efbbccea8a4719b6109572184c9d50_SeqId_4_] to archive 2024-12-09T03:48:49,548 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T03:48:49,550 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/1bb04daf1514486d89e40cf9017ac7dd to hdfs://localhost:35437/hbase/archive/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/1bb04daf1514486d89e40cf9017ac7dd 2024-12-09T03:48:49,551 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/5e7a90f29abe4ff4895241e25fcecfe8_SeqId_4_ to hdfs://localhost:35437/hbase/archive/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/5e7a90f29abe4ff4895241e25fcecfe8_SeqId_4_ 2024-12-09T03:48:49,552 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/87f5a3d7fcdb4b5b8f92627164030a6a_SeqId_4_ to hdfs://localhost:35437/hbase/archive/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/87f5a3d7fcdb4b5b8f92627164030a6a_SeqId_4_ 2024-12-09T03:48:49,553 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/a5efbbccea8a4719b6109572184c9d50_SeqId_4_ to hdfs://localhost:35437/hbase/archive/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/a/a5efbbccea8a4719b6109572184c9d50_SeqId_4_ 2024-12-09T03:48:49,553 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(327): Store=8105ced0dcc968d57f284af6c7939248/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:49,553 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:49,554 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8105ced0dcc968d57f284af6c7939248 columnFamilyName b 2024-12-09T03:48:49,554 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:49,554 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(327): Store=8105ced0dcc968d57f284af6c7939248/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:49,554 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:49,555 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8105ced0dcc968d57f284af6c7939248 columnFamilyName c 2024-12-09T03:48:49,555 DEBUG [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:49,556 INFO [StoreOpener-8105ced0dcc968d57f284af6c7939248-1 {}] regionserver.HStore(327): Store=8105ced0dcc968d57f284af6c7939248/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:49,556 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:49,556 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:49,558 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:49,558 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/recovered.edits/0000000000000000008 2024-12-09T03:48:49,585 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:49,587 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 8105ced0dcc968d57f284af6c7939248 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "8105ced0dcc968d57f284af6c7939248" family_name: "a" compaction_input: "1bb04daf1514486d89e40cf9017ac7dd" compaction_input: "87f5a3d7fcdb4b5b8f92627164030a6a_SeqId_4_" compaction_input: "5e7a90f29abe4ff4895241e25fcecfe8_SeqId_4_" compaction_input: "a5efbbccea8a4719b6109572184c9d50_SeqId_4_" compaction_output: "945e34006ac64a12b4daee40811db5d0" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-09T03:48:49,587 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-09T03:48:49,587 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/recovered.edits/0000000000000000008 2024-12-09T03:48:49,588 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/recovered.edits/0000000000000000008 2024-12-09T03:48:49,589 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:49,589 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:49,589 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:49,591 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8105ced0dcc968d57f284af6c7939248 2024-12-09T03:48:49,593 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testCompactedBulkLoadedFiles/8105ced0dcc968d57f284af6c7939248/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-09T03:48:49,593 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8105ced0dcc968d57f284af6c7939248; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73311374, jitterRate=0.09242460131645203}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:49,593 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8105ced0dcc968d57f284af6c7939248: Writing region info on filesystem at 1733716129524Initializing all the Stores at 1733716129525 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716129525Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716129525Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716129525Cleaning up temporary data from old regions at 1733716129589 (+64 ms)Region opened successfully at 1733716129593 (+4 ms) 2024-12-09T03:48:49,596 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8105ced0dcc968d57f284af6c7939248, disabling compactions & flushes 2024-12-09T03:48:49,596 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248. 2024-12-09T03:48:49,596 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248. 2024-12-09T03:48:49,596 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248. after waiting 0 ms 2024-12-09T03:48:49,596 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248. 2024-12-09T03:48:49,596 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733716125027.8105ced0dcc968d57f284af6c7939248. 2024-12-09T03:48:49,596 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8105ced0dcc968d57f284af6c7939248: Waiting for close lock at 1733716129595Disabling compacts and flushes for region at 1733716129595Disabling writes for close at 1733716129596 (+1 ms)Writing region close event to WAL at 1733716129596Closed at 1733716129596 2024-12-09T03:48:49,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741924_1103 (size=175) 2024-12-09T03:48:49,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741924_1103 (size=175) 2024-12-09T03:48:49,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741924_1103 (size=175) 2024-12-09T03:48:49,600 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T03:48:49,600 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733716129483) 2024-12-09T03:48:49,614 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=437 (was 423) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1139390664_22 at /127.0.0.1:52338 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35437 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:35437 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1139390664_22 at /127.0.0.1:37834 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1345 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=183 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=12020 (was 12065) 2024-12-09T03:48:49,615 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1345 is superior to 1024 2024-12-09T03:48:49,628 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=437, OpenFileDescriptor=1345, MaxFileDescriptor=1048576, SystemLoadAverage=183, ProcessCount=11, AvailableMemoryMB=12019 2024-12-09T03:48:49,628 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1345 is superior to 1024 2024-12-09T03:48:49,643 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:49,644 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:48:49,645 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:48:49,647 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-15012541, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/hregion-15012541, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:49,660 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-15012541/hregion-15012541.1733716129647, exclude list is [], retry=0 2024-12-09T03:48:49,663 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:49,663 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:49,663 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:49,690 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-15012541/hregion-15012541.1733716129647 2024-12-09T03:48:49,691 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:37281:37281)] 2024-12-09T03:48:49,691 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f2cf7eb4779a4270c16d45b0ea4ca3b3, NAME => 'testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35437/hbase 2024-12-09T03:48:49,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741926_1105 (size=67) 2024-12-09T03:48:49,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741926_1105 (size=67) 2024-12-09T03:48:49,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741926_1105 (size=67) 2024-12-09T03:48:49,700 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:49,701 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,702 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2cf7eb4779a4270c16d45b0ea4ca3b3 columnFamilyName a 2024-12-09T03:48:49,702 DEBUG [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:49,702 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(327): Store=f2cf7eb4779a4270c16d45b0ea4ca3b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:49,702 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,703 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2cf7eb4779a4270c16d45b0ea4ca3b3 columnFamilyName b 2024-12-09T03:48:49,704 DEBUG [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:49,704 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(327): Store=f2cf7eb4779a4270c16d45b0ea4ca3b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:49,704 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,705 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2cf7eb4779a4270c16d45b0ea4ca3b3 columnFamilyName c 2024-12-09T03:48:49,705 DEBUG [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:49,706 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(327): Store=f2cf7eb4779a4270c16d45b0ea4ca3b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:49,706 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,706 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,706 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,707 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,707 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,708 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:49,709 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,711 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:48:49,711 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f2cf7eb4779a4270c16d45b0ea4ca3b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60536213, jitterRate=-0.09794013202190399}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:49,711 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f2cf7eb4779a4270c16d45b0ea4ca3b3: Writing region info on filesystem at 1733716129700Initializing all the Stores at 1733716129700Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716129700Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716129701 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716129701Cleaning up temporary data from old regions at 1733716129707 (+6 ms)Region opened successfully at 1733716129711 (+4 ms) 2024-12-09T03:48:49,712 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f2cf7eb4779a4270c16d45b0ea4ca3b3, disabling compactions & flushes 2024-12-09T03:48:49,712 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. 2024-12-09T03:48:49,712 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. 2024-12-09T03:48:49,712 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. after waiting 0 ms 2024-12-09T03:48:49,712 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. 2024-12-09T03:48:49,712 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. 2024-12-09T03:48:49,712 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f2cf7eb4779a4270c16d45b0ea4ca3b3: Waiting for close lock at 1733716129712Disabling compacts and flushes for region at 1733716129712Disabling writes for close at 1733716129712Writing region close event to WAL at 1733716129712Closed at 1733716129712 2024-12-09T03:48:49,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741925_1104 (size=175) 2024-12-09T03:48:49,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741925_1104 (size=175) 2024-12-09T03:48:49,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741925_1104 (size=175) 2024-12-09T03:48:49,716 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T03:48:49,716 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-15012541:(num 1733716129647) 2024-12-09T03:48:49,716 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:49,718 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:49,731 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129718, exclude list is [], retry=0 2024-12-09T03:48:49,733 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:49,733 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:49,734 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:49,760 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129718 2024-12-09T03:48:49,760 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:38745:38745)] 2024-12-09T03:48:49,760 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f2cf7eb4779a4270c16d45b0ea4ca3b3, NAME => 'testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:49,760 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:49,760 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,760 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,762 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,762 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2cf7eb4779a4270c16d45b0ea4ca3b3 columnFamilyName a 2024-12-09T03:48:49,763 DEBUG [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:49,763 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(327): Store=f2cf7eb4779a4270c16d45b0ea4ca3b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:49,763 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,764 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2cf7eb4779a4270c16d45b0ea4ca3b3 columnFamilyName b 2024-12-09T03:48:49,764 DEBUG [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:49,764 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(327): Store=f2cf7eb4779a4270c16d45b0ea4ca3b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:49,764 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,765 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2cf7eb4779a4270c16d45b0ea4ca3b3 columnFamilyName c 2024-12-09T03:48:49,765 DEBUG [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:49,765 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(327): Store=f2cf7eb4779a4270c16d45b0ea4ca3b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:49,765 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,766 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,767 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,768 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,768 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,768 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:49,770 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,770 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f2cf7eb4779a4270c16d45b0ea4ca3b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66236692, jitterRate=-0.012996375560760498}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:49,771 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f2cf7eb4779a4270c16d45b0ea4ca3b3: Writing region info on filesystem at 1733716129760Initializing all the Stores at 1733716129761 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716129761Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716129762 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716129762Cleaning up temporary data from old regions at 1733716129768 (+6 ms)Region opened successfully at 1733716129770 (+2 ms) 2024-12-09T03:48:49,778 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-09T03:48:49,793 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/a/1f6d811f2bd741faac7492ab4c544414 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733716129771/Put/seqid=0 2024-12-09T03:48:49,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741928_1107 (size=5958) 2024-12-09T03:48:49,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741928_1107 (size=5958) 2024-12-09T03:48:49,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741928_1107 (size=5958) 2024-12-09T03:48:49,799 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/a/1f6d811f2bd741faac7492ab4c544414 2024-12-09T03:48:49,804 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/a/1f6d811f2bd741faac7492ab4c544414 as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/a/1f6d811f2bd741faac7492ab4c544414 2024-12-09T03:48:49,809 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/a/1f6d811f2bd741faac7492ab4c544414, entries=10, sequenceid=13, filesize=5.8 K 2024-12-09T03:48:49,810 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for f2cf7eb4779a4270c16d45b0ea4ca3b3 in 32ms, sequenceid=13, compaction requested=false 2024-12-09T03:48:49,810 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f2cf7eb4779a4270c16d45b0ea4ca3b3: 2024-12-09T03:48:49,826 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f2cf7eb4779a4270c16d45b0ea4ca3b3, disabling compactions & flushes 2024-12-09T03:48:49,826 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. 2024-12-09T03:48:49,826 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. 2024-12-09T03:48:49,826 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. after waiting 0 ms 2024-12-09T03:48:49,826 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. 2024-12-09T03:48:49,827 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. 2024-12-09T03:48:49,827 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. 2024-12-09T03:48:49,827 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f2cf7eb4779a4270c16d45b0ea4ca3b3: Waiting for close lock at 1733716129826Disabling compacts and flushes for region at 1733716129826Disabling writes for close at 1733716129826Writing region close event to WAL at 1733716129826Closed at 1733716129827 (+1 ms) 2024-12-09T03:48:49,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741927_1106 (size=6243) 2024-12-09T03:48:49,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741927_1106 (size=6243) 2024-12-09T03:48:49,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741927_1106 (size=6243) 2024-12-09T03:48:49,847 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129718, size=6.1 K (6243bytes) 2024-12-09T03:48:49,847 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129718 2024-12-09T03:48:49,847 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129718 after 0ms 2024-12-09T03:48:49,874 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:49,874 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129718 took 28ms 2024-12-09T03:48:49,877 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129718 so closing down 2024-12-09T03:48:49,877 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T03:48:49,878 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733716129718.temp 2024-12-09T03:48:49,903 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000003-wal.1733716129718.temp 2024-12-09T03:48:49,903 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T03:48:49,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741929_1108 (size=5575) 2024-12-09T03:48:49,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741929_1108 (size=5575) 2024-12-09T03:48:49,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741929_1108 (size=5575) 2024-12-09T03:48:49,911 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000003-wal.1733716129718.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T03:48:49,912 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000003-wal.1733716129718.temp to hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000035 2024-12-09T03:48:49,912 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 37 ms; skipped=2; WAL=hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129718, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-12-09T03:48:49,912 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129718, journal: Splitting hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129718, size=6.1 K (6243bytes) at 1733716129847Finishing writing output for hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129718 so closing down at 1733716129877 (+30 ms)Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000003-wal.1733716129718.temp at 1733716129903 (+26 ms)3 split writer threads finished at 1733716129903Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000003-wal.1733716129718.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733716129911 (+8 ms)Rename recovered edits hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000003-wal.1733716129718.temp to hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000035 at 1733716129912 (+1 ms)Processed 32 edits across 1 Regions in 37 ms; skipped=2; WAL=hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129718, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1733716129912 2024-12-09T03:48:49,914 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129718 to hdfs://localhost:35437/hbase/oldWALs/wal.1733716129718 2024-12-09T03:48:49,915 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000035 2024-12-09T03:48:49,915 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:49,916 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:49,929 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917, exclude list is [], retry=0 2024-12-09T03:48:49,932 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:49,932 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:49,933 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:49,959 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917 2024-12-09T03:48:49,960 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:36521:36521),(127.0.0.1/127.0.0.1:38745:38745)] 2024-12-09T03:48:49,960 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f2cf7eb4779a4270c16d45b0ea4ca3b3, NAME => 'testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:48:49,960 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:49,960 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,960 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,961 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,962 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2cf7eb4779a4270c16d45b0ea4ca3b3 columnFamilyName a 2024-12-09T03:48:49,962 DEBUG [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:49,967 DEBUG [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/a/1f6d811f2bd741faac7492ab4c544414 2024-12-09T03:48:49,967 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(327): Store=f2cf7eb4779a4270c16d45b0ea4ca3b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:49,967 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,968 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2cf7eb4779a4270c16d45b0ea4ca3b3 columnFamilyName b 2024-12-09T03:48:49,968 DEBUG [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:49,969 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(327): Store=f2cf7eb4779a4270c16d45b0ea4ca3b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:49,969 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,969 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2cf7eb4779a4270c16d45b0ea4ca3b3 columnFamilyName c 2024-12-09T03:48:49,969 DEBUG [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:49,970 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(327): Store=f2cf7eb4779a4270c16d45b0ea4ca3b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:49,970 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,970 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,972 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:49,972 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000035 2024-12-09T03:48:49,999 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:50,000 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000035 2024-12-09T03:48:50,000 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-09T03:48:50,014 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/b/988456843205464282a531db26a4fa66 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733716129810/Put/seqid=0 2024-12-09T03:48:50,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741931_1110 (size=5958) 2024-12-09T03:48:50,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741931_1110 (size=5958) 2024-12-09T03:48:50,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741931_1110 (size=5958) 2024-12-09T03:48:50,022 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/b/988456843205464282a531db26a4fa66 2024-12-09T03:48:50,042 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/c/b2f895d231ca4faaac6f9068a8d335cb is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733716129816/Put/seqid=0 2024-12-09T03:48:50,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741932_1111 (size=5958) 2024-12-09T03:48:50,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741932_1111 (size=5958) 2024-12-09T03:48:50,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741932_1111 (size=5958) 2024-12-09T03:48:50,049 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/c/b2f895d231ca4faaac6f9068a8d335cb 2024-12-09T03:48:50,055 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/b/988456843205464282a531db26a4fa66 as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/b/988456843205464282a531db26a4fa66 2024-12-09T03:48:50,060 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/b/988456843205464282a531db26a4fa66, entries=10, sequenceid=35, filesize=5.8 K 2024-12-09T03:48:50,061 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/c/b2f895d231ca4faaac6f9068a8d335cb as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/c/b2f895d231ca4faaac6f9068a8d335cb 2024-12-09T03:48:50,065 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/c/b2f895d231ca4faaac6f9068a8d335cb, entries=10, sequenceid=35, filesize=5.8 K 2024-12-09T03:48:50,066 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for f2cf7eb4779a4270c16d45b0ea4ca3b3 in 65ms, sequenceid=35, compaction requested=false; wal=null 2024-12-09T03:48:50,066 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000035 2024-12-09T03:48:50,067 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:50,067 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:50,067 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:50,069 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:50,070 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-09T03:48:50,071 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f2cf7eb4779a4270c16d45b0ea4ca3b3; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69470439, jitterRate=0.03519020974636078}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:50,072 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f2cf7eb4779a4270c16d45b0ea4ca3b3: Writing region info on filesystem at 1733716129960Initializing all the Stores at 1733716129961 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716129961Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716129961Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716129961Obtaining lock to block concurrent updates at 1733716130000 (+39 ms)Preparing flush snapshotting stores in f2cf7eb4779a4270c16d45b0ea4ca3b3 at 1733716130000Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733716130000Flushing stores of testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. at 1733716130000Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3/b: creating writer at 1733716130000Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3/b: appending metadata at 1733716130014 (+14 ms)Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3/b: closing flushed file at 1733716130014Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3/c: creating writer at 1733716130027 (+13 ms)Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3/c: appending metadata at 1733716130041 (+14 ms)Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3/c: closing flushed file at 1733716130042 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6241a8c4: reopening flushed file at 1733716130055 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78478242: reopening flushed file at 1733716130061 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for f2cf7eb4779a4270c16d45b0ea4ca3b3 in 65ms, sequenceid=35, compaction requested=false; wal=null at 1733716130066 (+5 ms)Cleaning up temporary data from old regions at 1733716130067 (+1 ms)Region opened successfully at 1733716130072 (+5 ms) 2024-12-09T03:48:50,137 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917, size=0 (0bytes) 2024-12-09T03:48:50,137 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917 might be still open, length is 0 2024-12-09T03:48:50,137 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917 2024-12-09T03:48:50,137 WARN [IPC Server handler 4 on default port 35437 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-09T03:48:50,137 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917 after 0ms 2024-12-09T03:48:51,144 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:60990 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:45815:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60990 dst: /127.0.0.1:45815 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45815 remote=/127.0.0.1:60990]. Total timeout mills is 60000, 58955 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:51,144 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:52418 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:39115:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52418 dst: /127.0.0.1:39115 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:51,144 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-107684321_22 at /127.0.0.1:37860 [Receiving block BP-2031904063-172.17.0.2-1733716090120:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:36217:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37860 dst: /127.0.0.1:36217 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:51,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741930_1112 (size=5567) 2024-12-09T03:48:51,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741930_1112 (size=5567) 2024-12-09T03:48:53,610 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-09T03:48:53,611 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-09T03:48:53,611 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-09T03:48:53,611 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-09T03:48:54,138 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917 after 4001ms 2024-12-09T03:48:54,168 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:54,168 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917 took 4031ms 2024-12-09T03:48:54,170 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917; continuing. 2024-12-09T03:48:54,170 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917 so closing down 2024-12-09T03:48:54,170 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T03:48:54,172 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733716129917.temp 2024-12-09T03:48:54,198 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000037-wal.1733716129917.temp 2024-12-09T03:48:54,199 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T03:48:54,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741933_1113 (size=5575) 2024-12-09T03:48:54,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741933_1113 (size=5575) 2024-12-09T03:48:54,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741933_1113 (size=5575) 2024-12-09T03:48:54,206 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000037-wal.1733716129917.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T03:48:54,207 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000037-wal.1733716129917.temp to hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000066 2024-12-09T03:48:54,207 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 38 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T03:48:54,207 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917, journal: Splitting hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917, size=0 (0bytes) at 1733716130137Finishing writing output for hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917 so closing down at 1733716134170 (+4033 ms)Creating recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000037-wal.1733716129917.temp at 1733716134198 (+28 ms)3 split writer threads finished at 1733716134199 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000037-wal.1733716129917.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733716134206 (+7 ms)Rename recovered edits hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000037-wal.1733716129917.temp to hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000066 at 1733716134207 (+1 ms)Processed 30 edits across 1 Regions in 38 ms; skipped=0; WAL=hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917, size=0, length=0, corrupted=false, cancelled=false at 1733716134207 2024-12-09T03:48:54,208 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917 to hdfs://localhost:35437/hbase/oldWALs/wal.1733716129917 2024-12-09T03:48:54,209 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000066 2024-12-09T03:48:54,209 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T03:48:54,211 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35437/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642, archiveDir=hdfs://localhost:35437/hbase/oldWALs, maxLogs=32 2024-12-09T03:48:54,224 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716134211, exclude list is [], retry=0 2024-12-09T03:48:54,227 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-08197379-152b-4313-b695-1141a205ab89,DISK] 2024-12-09T03:48:54,227 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39115,DS-21e3b170-8d5d-42d3-83cd-217896bbed1f,DISK] 2024-12-09T03:48:54,227 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36217,DS-01cd25b1-970b-4ee3-a2f2-b09a0016e27d,DISK] 2024-12-09T03:48:54,255 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716134211 2024-12-09T03:48:54,255 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37281:37281),(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:36521:36521)] 2024-12-09T03:48:54,256 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:48:54,257 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:54,258 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2cf7eb4779a4270c16d45b0ea4ca3b3 columnFamilyName a 2024-12-09T03:48:54,258 DEBUG [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:54,263 DEBUG [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/a/1f6d811f2bd741faac7492ab4c544414 2024-12-09T03:48:54,263 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(327): Store=f2cf7eb4779a4270c16d45b0ea4ca3b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:54,263 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:54,264 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2cf7eb4779a4270c16d45b0ea4ca3b3 columnFamilyName b 2024-12-09T03:48:54,264 DEBUG [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:54,268 DEBUG [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/b/988456843205464282a531db26a4fa66 2024-12-09T03:48:54,268 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(327): Store=f2cf7eb4779a4270c16d45b0ea4ca3b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:54,268 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:54,269 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2cf7eb4779a4270c16d45b0ea4ca3b3 columnFamilyName c 2024-12-09T03:48:54,269 DEBUG [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:48:54,273 DEBUG [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/c/b2f895d231ca4faaac6f9068a8d335cb 2024-12-09T03:48:54,273 INFO [StoreOpener-f2cf7eb4779a4270c16d45b0ea4ca3b3-1 {}] regionserver.HStore(327): Store=f2cf7eb4779a4270c16d45b0ea4ca3b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:48:54,273 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:54,274 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:54,275 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:54,275 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000066 2024-12-09T03:48:54,302 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T03:48:54,308 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000066 2024-12-09T03:48:54,308 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-09T03:48:54,323 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/a/82bd21da208145ecb2478f1e483730f3 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733716130078/Put/seqid=0 2024-12-09T03:48:54,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741935_1115 (size=5958) 2024-12-09T03:48:54,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741935_1115 (size=5958) 2024-12-09T03:48:54,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741935_1115 (size=5958) 2024-12-09T03:48:54,330 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/a/82bd21da208145ecb2478f1e483730f3 2024-12-09T03:48:54,351 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/b/f2d0df0775a140f78758132df41fc2ec is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733716130085/Put/seqid=0 2024-12-09T03:48:54,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741936_1116 (size=5958) 2024-12-09T03:48:54,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741936_1116 (size=5958) 2024-12-09T03:48:54,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741936_1116 (size=5958) 2024-12-09T03:48:54,358 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/b/f2d0df0775a140f78758132df41fc2ec 2024-12-09T03:48:54,377 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/c/e581a97752bf448d9ddcf4cb70da70e2 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733716130092/Put/seqid=0 2024-12-09T03:48:54,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741937_1117 (size=5958) 2024-12-09T03:48:54,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741937_1117 (size=5958) 2024-12-09T03:48:54,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741937_1117 (size=5958) 2024-12-09T03:48:54,383 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/c/e581a97752bf448d9ddcf4cb70da70e2 2024-12-09T03:48:54,388 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/a/82bd21da208145ecb2478f1e483730f3 as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/a/82bd21da208145ecb2478f1e483730f3 2024-12-09T03:48:54,392 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/a/82bd21da208145ecb2478f1e483730f3, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T03:48:54,393 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/b/f2d0df0775a140f78758132df41fc2ec as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/b/f2d0df0775a140f78758132df41fc2ec 2024-12-09T03:48:54,396 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/b/f2d0df0775a140f78758132df41fc2ec, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T03:48:54,397 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/.tmp/c/e581a97752bf448d9ddcf4cb70da70e2 as hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/c/e581a97752bf448d9ddcf4cb70da70e2 2024-12-09T03:48:54,400 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/c/e581a97752bf448d9ddcf4cb70da70e2, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T03:48:54,401 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for f2cf7eb4779a4270c16d45b0ea4ca3b3 in 93ms, sequenceid=66, compaction requested=false; wal=null 2024-12-09T03:48:54,401 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/0000000000000000066 2024-12-09T03:48:54,402 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:54,402 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:54,402 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:48:54,403 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f2cf7eb4779a4270c16d45b0ea4ca3b3 2024-12-09T03:48:54,405 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/hbase/data/default/testReplayEditsWrittenViaHRegion/f2cf7eb4779a4270c16d45b0ea4ca3b3/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-09T03:48:54,406 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f2cf7eb4779a4270c16d45b0ea4ca3b3; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70368893, jitterRate=0.04857821762561798}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:48:54,406 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f2cf7eb4779a4270c16d45b0ea4ca3b3: Writing region info on filesystem at 1733716134256Initializing all the Stores at 1733716134257 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716134257Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716134257Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733716134257Obtaining lock to block concurrent updates at 1733716134308 (+51 ms)Preparing flush snapshotting stores in f2cf7eb4779a4270c16d45b0ea4ca3b3 at 1733716134308Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733716134308Flushing stores of testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. at 1733716134308Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3/a: creating writer at 1733716134308Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3/a: appending metadata at 1733716134322 (+14 ms)Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3/a: closing flushed file at 1733716134322Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3/b: creating writer at 1733716134336 (+14 ms)Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3/b: appending metadata at 1733716134350 (+14 ms)Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3/b: closing flushed file at 1733716134350Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3/c: creating writer at 1733716134363 (+13 ms)Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3/c: appending metadata at 1733716134376 (+13 ms)Flushing f2cf7eb4779a4270c16d45b0ea4ca3b3/c: closing flushed file at 1733716134376Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f24066a: reopening flushed file at 1733716134387 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cd36cf7: reopening flushed file at 1733716134392 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29aefdfa: reopening flushed file at 1733716134396 (+4 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for f2cf7eb4779a4270c16d45b0ea4ca3b3 in 93ms, sequenceid=66, compaction requested=false; wal=null at 1733716134401 (+5 ms)Cleaning up temporary data from old regions at 1733716134402 (+1 ms)Region opened successfully at 1733716134406 (+4 ms) 2024-12-09T03:48:54,420 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f2cf7eb4779a4270c16d45b0ea4ca3b3, disabling compactions & flushes 2024-12-09T03:48:54,420 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. 2024-12-09T03:48:54,420 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. 2024-12-09T03:48:54,420 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. after waiting 0 ms 2024-12-09T03:48:54,420 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. 2024-12-09T03:48:54,422 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733716129643.f2cf7eb4779a4270c16d45b0ea4ca3b3. 2024-12-09T03:48:54,422 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f2cf7eb4779a4270c16d45b0ea4ca3b3: Waiting for close lock at 1733716134420Disabling compacts and flushes for region at 1733716134420Disabling writes for close at 1733716134420Writing region close event to WAL at 1733716134421 (+1 ms)Closed at 1733716134421 2024-12-09T03:48:54,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741934_1114 (size=175) 2024-12-09T03:48:54,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741934_1114 (size=175) 2024-12-09T03:48:54,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741934_1114 (size=175) 2024-12-09T03:48:54,427 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T03:48:54,427 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733716134211) 2024-12-09T03:48:54,442 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=444 (was 437) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35437 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1035041931_22 at /127.0.0.1:38724 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1035041931_22 at /127.0.0.1:39056 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1035041931_22 at /127.0.0.1:33550 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:35437 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1412 (was 1345) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=168 (was 183), ProcessCount=11 (was 11), AvailableMemoryMB=12005 (was 12019) 2024-12-09T03:48:54,442 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1412 is superior to 1024 2024-12-09T03:48:54,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T03:48:54,443 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T03:48:54,443 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T03:48:54,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:48:54,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:48:54,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T03:48:54,444 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T03:48:54,444 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=938327945, stopped=false 2024-12-09T03:48:54,444 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=55d0183f16d2,34431,1733716093194 2024-12-09T03:48:54,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T03:48:54,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:54,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T03:48:54,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:54,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T03:48:54,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:54,446 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T03:48:54,447 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T03:48:54,447 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T03:48:54,447 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T03:48:54,447 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T03:48:54,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:48:54,447 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T03:48:54,448 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '55d0183f16d2,39859,1733716093905' ***** 2024-12-09T03:48:54,448 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T03:48:54,448 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '55d0183f16d2,35501,1733716094084' ***** 2024-12-09T03:48:54,448 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T03:48:54,448 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T03:48:54,448 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T03:48:54,448 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T03:48:54,448 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T03:48:54,448 INFO [RS:0;55d0183f16d2:39859 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T03:48:54,448 INFO [RS:0;55d0183f16d2:39859 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T03:48:54,448 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.HRegionServer(959): stopping server 55d0183f16d2,39859,1733716093905 2024-12-09T03:48:54,449 INFO [RS:0;55d0183f16d2:39859 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T03:48:54,449 INFO [RS:0;55d0183f16d2:39859 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;55d0183f16d2:39859. 2024-12-09T03:48:54,449 DEBUG [RS:0;55d0183f16d2:39859 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T03:48:54,449 DEBUG [RS:0;55d0183f16d2:39859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:48:54,449 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.HRegionServer(976): stopping server 55d0183f16d2,39859,1733716093905; all regions closed. 2024-12-09T03:48:54,449 INFO [RS:2;55d0183f16d2:35501 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T03:48:54,449 INFO [RS:2;55d0183f16d2:35501 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T03:48:54,449 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(3091): Received CLOSE for 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:54,449 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(959): stopping server 55d0183f16d2,35501,1733716094084 2024-12-09T03:48:54,449 INFO [RS:2;55d0183f16d2:35501 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T03:48:54,449 INFO [RS:2;55d0183f16d2:35501 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;55d0183f16d2:35501. 2024-12-09T03:48:54,450 DEBUG [RS:2;55d0183f16d2:35501 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T03:48:54,450 DEBUG [RS:2;55d0183f16d2:35501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:48:54,450 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 940a5def324c1e7ca6d1994726e09abd, disabling compactions & flushes 2024-12-09T03:48:54,450 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:54,450 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T03:48:54,450 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T03:48:54,450 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:54,450 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T03:48:54,450 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. after waiting 0 ms 2024-12-09T03:48:54,450 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:54,450 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T03:48:54,450 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-09T03:48:54,450 DEBUG [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 940a5def324c1e7ca6d1994726e09abd=testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd.} 2024-12-09T03:48:54,450 DEBUG [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 940a5def324c1e7ca6d1994726e09abd 2024-12-09T03:48:54,450 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T03:48:54,451 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T03:48:54,451 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T03:48:54,451 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T03:48:54,451 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T03:48:54,451 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-09T03:48:54,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741835_1011 (size=2764) 2024-12-09T03:48:54,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741835_1011 (size=2764) 2024-12-09T03:48:54,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741835_1011 (size=2764) 2024-12-09T03:48:54,455 DEBUG [RS:0;55d0183f16d2:39859 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/oldWALs 2024-12-09T03:48:54,455 INFO [RS:0;55d0183f16d2:39859 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 55d0183f16d2%2C39859%2C1733716093905:(num 1733716095554) 2024-12-09T03:48:54,455 DEBUG [RS:0;55d0183f16d2:39859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:48:54,455 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T03:48:54,455 INFO [RS:0;55d0183f16d2:39859 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T03:48:54,456 INFO [RS:0;55d0183f16d2:39859 {}] hbase.ChoreService(370): Chore service for: regionserver/55d0183f16d2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T03:48:54,456 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T03:48:54,456 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T03:48:54,456 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T03:48:54,456 INFO [regionserver/55d0183f16d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T03:48:54,456 INFO [RS:0;55d0183f16d2:39859 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T03:48:54,456 INFO [RS:0;55d0183f16d2:39859 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39859 2024-12-09T03:48:54,456 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/default/testReplayEditsAfterRegionMovedWithMultiCF/940a5def324c1e7ca6d1994726e09abd/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-09T03:48:54,457 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:54,458 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 940a5def324c1e7ca6d1994726e09abd: Waiting for close lock at 1733716134450Running coprocessor pre-close hooks at 1733716134450Disabling compacts and flushes for region at 1733716134450Disabling writes for close at 1733716134450Writing region close event to WAL at 1733716134452 (+2 ms)Running coprocessor post-close hooks at 1733716134457 (+5 ms)Closed at 1733716134457 2024-12-09T03:48:54,458 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd. 2024-12-09T03:48:54,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/55d0183f16d2,39859,1733716093905 2024-12-09T03:48:54,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T03:48:54,458 INFO [RS:0;55d0183f16d2:39859 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T03:48:54,459 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [55d0183f16d2,39859,1733716093905] 2024-12-09T03:48:54,462 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/55d0183f16d2,39859,1733716093905 already deleted, retry=false 2024-12-09T03:48:54,462 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 55d0183f16d2,39859,1733716093905 expired; onlineServers=1 2024-12-09T03:48:54,480 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740/.tmp/info/0a474a24bf1b434a811ede30cb2c838f is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733716109530.940a5def324c1e7ca6d1994726e09abd./info:regioninfo/1733716113695/Put/seqid=0 2024-12-09T03:48:54,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741938_1118 (size=8243) 2024-12-09T03:48:54,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741938_1118 (size=8243) 2024-12-09T03:48:54,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741938_1118 (size=8243) 2024-12-09T03:48:54,489 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740/.tmp/info/0a474a24bf1b434a811ede30cb2c838f 2024-12-09T03:48:54,509 INFO [regionserver/55d0183f16d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T03:48:54,511 INFO [regionserver/55d0183f16d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T03:48:54,512 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740/.tmp/ns/e54fc2db307e407394b2745f1216b788 is 43, key is default/ns:d/1733716096080/Put/seqid=0 2024-12-09T03:48:54,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741939_1119 (size=5153) 2024-12-09T03:48:54,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741939_1119 (size=5153) 2024-12-09T03:48:54,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741939_1119 (size=5153) 2024-12-09T03:48:54,519 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740/.tmp/ns/e54fc2db307e407394b2745f1216b788 2024-12-09T03:48:54,540 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740/.tmp/table/64f014aaf4c047959ac3c90cf9348a46 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733716110419/Put/seqid=0 2024-12-09T03:48:54,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741940_1120 (size=5431) 2024-12-09T03:48:54,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741940_1120 (size=5431) 2024-12-09T03:48:54,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741940_1120 (size=5431) 2024-12-09T03:48:54,547 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740/.tmp/table/64f014aaf4c047959ac3c90cf9348a46 2024-12-09T03:48:54,554 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740/.tmp/info/0a474a24bf1b434a811ede30cb2c838f as hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740/info/0a474a24bf1b434a811ede30cb2c838f 2024-12-09T03:48:54,559 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740/info/0a474a24bf1b434a811ede30cb2c838f, entries=18, sequenceid=21, filesize=8.0 K 2024-12-09T03:48:54,560 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740/.tmp/ns/e54fc2db307e407394b2745f1216b788 as hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740/ns/e54fc2db307e407394b2745f1216b788 2024-12-09T03:48:54,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:48:54,560 INFO [RS:0;55d0183f16d2:39859 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T03:48:54,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39859-0x1019556106d0001, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:48:54,560 INFO [RS:0;55d0183f16d2:39859 {}] regionserver.HRegionServer(1031): Exiting; stopping=55d0183f16d2,39859,1733716093905; zookeeper connection closed. 2024-12-09T03:48:54,561 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@62a0e6e2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@62a0e6e2 2024-12-09T03:48:54,565 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740/ns/e54fc2db307e407394b2745f1216b788, entries=2, sequenceid=21, filesize=5.0 K 2024-12-09T03:48:54,566 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740/.tmp/table/64f014aaf4c047959ac3c90cf9348a46 as hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740/table/64f014aaf4c047959ac3c90cf9348a46 2024-12-09T03:48:54,570 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740/table/64f014aaf4c047959ac3c90cf9348a46, entries=2, sequenceid=21, filesize=5.3 K 2024-12-09T03:48:54,571 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 120ms, sequenceid=21, compaction requested=false 2024-12-09T03:48:54,575 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-09T03:48:54,576 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T03:48:54,576 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T03:48:54,576 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733716134450Running coprocessor pre-close hooks at 1733716134450Disabling compacts and flushes for region at 1733716134450Disabling writes for close at 1733716134451 (+1 ms)Obtaining lock to block concurrent updates at 1733716134451Preparing flush snapshotting stores in 1588230740 at 1733716134451Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733716134452 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733716134457 (+5 ms)Flushing 1588230740/info: creating writer at 1733716134457Flushing 1588230740/info: appending metadata at 1733716134479 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733716134479Flushing 1588230740/ns: creating writer at 1733716134495 (+16 ms)Flushing 1588230740/ns: appending metadata at 1733716134511 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733716134511Flushing 1588230740/table: creating writer at 1733716134525 (+14 ms)Flushing 1588230740/table: appending metadata at 1733716134539 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733716134540 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4416146a: reopening flushed file at 1733716134553 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dd7e05b: reopening flushed file at 1733716134559 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@98e3b80: reopening flushed file at 1733716134565 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 120ms, sequenceid=21, compaction requested=false at 1733716134571 (+6 ms)Writing region close event to WAL at 1733716134572 (+1 ms)Running coprocessor post-close hooks at 1733716134576 (+4 ms)Closed at 1733716134576 2024-12-09T03:48:54,576 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T03:48:54,651 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(976): stopping server 55d0183f16d2,35501,1733716094084; all regions closed. 2024-12-09T03:48:54,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741836_1012 (size=9487) 2024-12-09T03:48:54,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741836_1012 (size=9487) 2024-12-09T03:48:54,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741836_1012 (size=9487) 2024-12-09T03:48:54,656 DEBUG [RS:2;55d0183f16d2:35501 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/oldWALs 2024-12-09T03:48:54,656 INFO [RS:2;55d0183f16d2:35501 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 55d0183f16d2%2C35501%2C1733716094084.meta:.meta(num 1733716095879) 2024-12-09T03:48:54,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741833_1009 (size=1032) 2024-12-09T03:48:54,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741833_1009 (size=1032) 2024-12-09T03:48:54,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741833_1009 (size=1032) 2024-12-09T03:48:54,659 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/WALs/55d0183f16d2,35501,1733716094084/55d0183f16d2%2C35501%2C1733716094084.1733716095554 not finished, retry = 0 2024-12-09T03:48:54,761 DEBUG [RS:2;55d0183f16d2:35501 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/oldWALs 2024-12-09T03:48:54,761 INFO [RS:2;55d0183f16d2:35501 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 55d0183f16d2%2C35501%2C1733716094084:(num 1733716095554) 2024-12-09T03:48:54,762 DEBUG [RS:2;55d0183f16d2:35501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:48:54,762 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T03:48:54,762 INFO [RS:2;55d0183f16d2:35501 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T03:48:54,762 INFO [RS:2;55d0183f16d2:35501 {}] hbase.ChoreService(370): Chore service for: regionserver/55d0183f16d2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T03:48:54,762 INFO [RS:2;55d0183f16d2:35501 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T03:48:54,762 INFO [regionserver/55d0183f16d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T03:48:54,762 INFO [RS:2;55d0183f16d2:35501 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35501 2024-12-09T03:48:54,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T03:48:54,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/55d0183f16d2,35501,1733716094084 2024-12-09T03:48:54,764 INFO [RS:2;55d0183f16d2:35501 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T03:48:54,765 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [55d0183f16d2,35501,1733716094084] 2024-12-09T03:48:54,767 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/55d0183f16d2,35501,1733716094084 already deleted, retry=false 2024-12-09T03:48:54,767 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 55d0183f16d2,35501,1733716094084 expired; onlineServers=0 2024-12-09T03:48:54,767 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '55d0183f16d2,34431,1733716093194' ***** 2024-12-09T03:48:54,767 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T03:48:54,767 INFO [M:0;55d0183f16d2:34431 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T03:48:54,767 INFO [M:0;55d0183f16d2:34431 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T03:48:54,767 DEBUG [M:0;55d0183f16d2:34431 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T03:48:54,767 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T03:48:54,767 DEBUG [M:0;55d0183f16d2:34431 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T03:48:54,767 DEBUG [master/55d0183f16d2:0:becomeActiveMaster-HFileCleaner.small.0-1733716095114 {}] cleaner.HFileCleaner(306): Exit Thread[master/55d0183f16d2:0:becomeActiveMaster-HFileCleaner.small.0-1733716095114,5,FailOnTimeoutGroup] 2024-12-09T03:48:54,767 DEBUG [master/55d0183f16d2:0:becomeActiveMaster-HFileCleaner.large.0-1733716095113 {}] cleaner.HFileCleaner(306): Exit Thread[master/55d0183f16d2:0:becomeActiveMaster-HFileCleaner.large.0-1733716095113,5,FailOnTimeoutGroup] 2024-12-09T03:48:54,768 INFO [M:0;55d0183f16d2:34431 {}] hbase.ChoreService(370): Chore service for: master/55d0183f16d2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T03:48:54,768 INFO [M:0;55d0183f16d2:34431 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T03:48:54,768 DEBUG [M:0;55d0183f16d2:34431 {}] master.HMaster(1795): Stopping service threads 2024-12-09T03:48:54,768 INFO [M:0;55d0183f16d2:34431 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T03:48:54,768 INFO [M:0;55d0183f16d2:34431 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T03:48:54,769 INFO [M:0;55d0183f16d2:34431 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T03:48:54,769 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T03:48:54,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T03:48:54,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:48:54,771 DEBUG [M:0;55d0183f16d2:34431 {}] zookeeper.ZKUtil(347): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T03:48:54,771 WARN [M:0;55d0183f16d2:34431 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T03:48:54,772 INFO [M:0;55d0183f16d2:34431 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/.lastflushedseqids 2024-12-09T03:48:54,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741941_1121 (size=138) 2024-12-09T03:48:54,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741941_1121 (size=138) 2024-12-09T03:48:54,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741941_1121 (size=138) 2024-12-09T03:48:54,784 INFO [M:0;55d0183f16d2:34431 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T03:48:54,784 INFO [M:0;55d0183f16d2:34431 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T03:48:54,785 DEBUG [M:0;55d0183f16d2:34431 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T03:48:54,785 INFO [M:0;55d0183f16d2:34431 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:48:54,785 DEBUG [M:0;55d0183f16d2:34431 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:48:54,785 DEBUG [M:0;55d0183f16d2:34431 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T03:48:54,785 DEBUG [M:0;55d0183f16d2:34431 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:48:54,785 INFO [M:0;55d0183f16d2:34431 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.33 KB heapSize=83.72 KB 2024-12-09T03:48:54,800 DEBUG [M:0;55d0183f16d2:34431 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7c1787a3d24446b48b0e69f558abdbae is 82, key is hbase:meta,,1/info:regioninfo/1733716096004/Put/seqid=0 2024-12-09T03:48:54,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741942_1122 (size=5672) 2024-12-09T03:48:54,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741942_1122 (size=5672) 2024-12-09T03:48:54,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741942_1122 (size=5672) 2024-12-09T03:48:54,806 INFO [M:0;55d0183f16d2:34431 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7c1787a3d24446b48b0e69f558abdbae 2024-12-09T03:48:54,825 DEBUG [M:0;55d0183f16d2:34431 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4ce500bbd2bd4708b93dd5b0ecd783f2 is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733716110425/Put/seqid=0 2024-12-09T03:48:54,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741943_1123 (size=7754) 2024-12-09T03:48:54,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741943_1123 (size=7754) 2024-12-09T03:48:54,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741943_1123 (size=7754) 2024-12-09T03:48:54,832 INFO [M:0;55d0183f16d2:34431 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4ce500bbd2bd4708b93dd5b0ecd783f2 2024-12-09T03:48:54,837 INFO [M:0;55d0183f16d2:34431 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4ce500bbd2bd4708b93dd5b0ecd783f2 2024-12-09T03:48:54,852 DEBUG [M:0;55d0183f16d2:34431 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/99d98a8014544ff7961df447fb2046b9 is 69, key is 55d0183f16d2,35501,1733716094084/rs:state/1733716095280/Put/seqid=0 2024-12-09T03:48:54,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741944_1124 (size=5440) 2024-12-09T03:48:54,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741944_1124 (size=5440) 2024-12-09T03:48:54,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741944_1124 (size=5440) 2024-12-09T03:48:54,863 INFO [M:0;55d0183f16d2:34431 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/99d98a8014544ff7961df447fb2046b9 2024-12-09T03:48:54,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:48:54,866 INFO [RS:2;55d0183f16d2:35501 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T03:48:54,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35501-0x1019556106d0003, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:48:54,866 INFO [RS:2;55d0183f16d2:35501 {}] regionserver.HRegionServer(1031): Exiting; stopping=55d0183f16d2,35501,1733716094084; zookeeper connection closed. 2024-12-09T03:48:54,867 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4674a361 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4674a361 2024-12-09T03:48:54,867 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-09T03:48:54,869 INFO [M:0;55d0183f16d2:34431 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 99d98a8014544ff7961df447fb2046b9 2024-12-09T03:48:54,870 DEBUG [M:0;55d0183f16d2:34431 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7c1787a3d24446b48b0e69f558abdbae as hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7c1787a3d24446b48b0e69f558abdbae 2024-12-09T03:48:54,875 INFO [M:0;55d0183f16d2:34431 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7c1787a3d24446b48b0e69f558abdbae, entries=8, sequenceid=168, filesize=5.5 K 2024-12-09T03:48:54,876 DEBUG [M:0;55d0183f16d2:34431 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4ce500bbd2bd4708b93dd5b0ecd783f2 as hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4ce500bbd2bd4708b93dd5b0ecd783f2 2024-12-09T03:48:54,882 INFO [M:0;55d0183f16d2:34431 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4ce500bbd2bd4708b93dd5b0ecd783f2 2024-12-09T03:48:54,882 INFO [M:0;55d0183f16d2:34431 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4ce500bbd2bd4708b93dd5b0ecd783f2, entries=17, sequenceid=168, filesize=7.6 K 2024-12-09T03:48:54,883 DEBUG [M:0;55d0183f16d2:34431 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/99d98a8014544ff7961df447fb2046b9 as hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/99d98a8014544ff7961df447fb2046b9 2024-12-09T03:48:54,888 INFO [M:0;55d0183f16d2:34431 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 99d98a8014544ff7961df447fb2046b9 2024-12-09T03:48:54,888 INFO [M:0;55d0183f16d2:34431 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/0427f071-c332-24d0-0042-d7ca532e1bb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/99d98a8014544ff7961df447fb2046b9, entries=3, sequenceid=168, filesize=5.3 K 2024-12-09T03:48:54,889 INFO [M:0;55d0183f16d2:34431 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.33 KB/69972, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=168, compaction requested=false 2024-12-09T03:48:54,890 INFO [M:0;55d0183f16d2:34431 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:48:54,890 DEBUG [M:0;55d0183f16d2:34431 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733716134785Disabling compacts and flushes for region at 1733716134785Disabling writes for close at 1733716134785Obtaining lock to block concurrent updates at 1733716134785Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733716134785Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69972, getHeapSize=85664, getOffHeapSize=0, getCellsCount=195 at 1733716134785Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733716134786 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733716134786Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733716134799 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733716134800 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733716134810 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733716134824 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733716134824Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733716134837 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733716134852 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733716134852Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f0a2a6f: reopening flushed file at 1733716134869 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ced74fb: reopening flushed file at 1733716134875 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6aa6ca65: reopening flushed file at 1733716134882 (+7 ms)Finished flush of dataSize ~68.33 KB/69972, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=168, compaction requested=false at 1733716134889 (+7 ms)Writing region close event to WAL at 1733716134890 (+1 ms)Closed at 1733716134890 2024-12-09T03:48:54,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39115 is added to blk_1073741830_1006 (size=83594) 2024-12-09T03:48:54,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36217 is added to blk_1073741830_1006 (size=83594) 2024-12-09T03:48:54,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741830_1006 (size=83594) 2024-12-09T03:48:54,894 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T03:48:54,894 INFO [M:0;55d0183f16d2:34431 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T03:48:54,895 INFO [M:0;55d0183f16d2:34431 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34431 2024-12-09T03:48:54,895 INFO [M:0;55d0183f16d2:34431 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T03:48:54,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:48:54,997 INFO [M:0;55d0183f16d2:34431 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T03:48:54,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34431-0x1019556106d0000, quorum=127.0.0.1:62749, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:48:55,004 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114450 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114450 (inode 16655) Holder DFSClient_NONMAPREDUCE_-107684321_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733716114036/wal.1733716114450 (inode 16655) Holder DFSClient_NONMAPREDUCE_-107684321_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-09T03:48:55,005 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733716104837/wal.1733716105002 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:55,005 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733716129642/wal.1733716129917 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:55,007 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716123998 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716123998 (inode 16678) Holder DFSClient_NONMAPREDUCE_-107684321_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733716114639/wal.1733716123998 (inode 16678) Holder DFSClient_NONMAPREDUCE_-107684321_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-09T03:48:55,008 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733716097887/wal.1733716097988 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:55,010 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124828 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124828 (inode 16704) Holder DFSClient_NONMAPREDUCE_-107684321_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733716124253/wal.1733716124828 (inode 16704) Holder DFSClient_NONMAPREDUCE_-107684321_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-09T03:48:55,011 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733716125025/wal.1733716125112 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:48:55,014 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal.1733716097698 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal.1733716097698 (inode 16485) Holder DFSClient_NONMAPREDUCE_-107684321_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733716097277/wal.1733716097698 (inode 16485) Holder DFSClient_NONMAPREDUCE_-107684321_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-09T03:48:55,016 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal.1733716096927 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal.1733716096927 (inode 16462) Holder DFSClient_NONMAPREDUCE_-107684321_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733716096330/wal.1733716096927 (inode 16462) Holder DFSClient_NONMAPREDUCE_-107684321_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-09T03:48:55,021 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T03:48:55,023 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T03:48:55,024 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T03:48:55,024 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T03:48:55,024 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/hadoop.log.dir/,STOPPED} 2024-12-09T03:48:55,027 WARN [BP-2031904063-172.17.0.2-1733716090120 heartbeating to localhost/127.0.0.1:35437 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T03:48:55,027 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T03:48:55,027 WARN [BP-2031904063-172.17.0.2-1733716090120 heartbeating to localhost/127.0.0.1:35437 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2031904063-172.17.0.2-1733716090120 (Datanode Uuid a349d05a-2373-4b67-812b-e88082e5940a) service to localhost/127.0.0.1:35437 2024-12-09T03:48:55,027 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T03:48:55,029 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data5/current/BP-2031904063-172.17.0.2-1733716090120 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T03:48:55,029 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data6/current/BP-2031904063-172.17.0.2-1733716090120 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T03:48:55,030 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T03:48:55,032 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T03:48:55,032 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T03:48:55,032 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T03:48:55,032 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T03:48:55,032 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/hadoop.log.dir/,STOPPED} 2024-12-09T03:48:55,034 WARN [BP-2031904063-172.17.0.2-1733716090120 heartbeating to localhost/127.0.0.1:35437 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T03:48:55,034 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T03:48:55,034 WARN [BP-2031904063-172.17.0.2-1733716090120 heartbeating to localhost/127.0.0.1:35437 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2031904063-172.17.0.2-1733716090120 (Datanode Uuid 40d5c32f-369d-4872-94b6-2f2d7378e3e0) service to localhost/127.0.0.1:35437 2024-12-09T03:48:55,034 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T03:48:55,035 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data3/current/BP-2031904063-172.17.0.2-1733716090120 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T03:48:55,035 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data4/current/BP-2031904063-172.17.0.2-1733716090120 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T03:48:55,035 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T03:48:55,037 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T03:48:55,038 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T03:48:55,038 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T03:48:55,038 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T03:48:55,038 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/hadoop.log.dir/,STOPPED} 2024-12-09T03:48:55,039 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T03:48:55,039 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T03:48:55,039 WARN [BP-2031904063-172.17.0.2-1733716090120 heartbeating to localhost/127.0.0.1:35437 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T03:48:55,039 WARN [BP-2031904063-172.17.0.2-1733716090120 heartbeating to localhost/127.0.0.1:35437 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2031904063-172.17.0.2-1733716090120 (Datanode Uuid 796afa67-5d59-4e50-bcd5-6fb248bd28eb) service to localhost/127.0.0.1:35437 2024-12-09T03:48:55,040 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data1/current/BP-2031904063-172.17.0.2-1733716090120 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T03:48:55,040 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/cluster_62e93f40-ef45-074c-8b91-484e3f616acc/data/data2/current/BP-2031904063-172.17.0.2-1733716090120 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T03:48:55,040 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T03:48:55,050 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T03:48:55,050 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T03:48:55,051 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T03:48:55,051 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T03:48:55,051 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a54614ce-6c15-1a47-8831-6f71b327bbae/hadoop.log.dir/,STOPPED} 2024-12-09T03:48:55,060 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T03:48:55,106 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down