2024-12-02 16:22:45,869 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-02 16:22:45,880 main DEBUG Took 0.008759 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-02 16:22:45,880 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-02 16:22:45,880 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-02 16:22:45,881 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-02 16:22:45,882 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 16:22:45,889 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-02 16:22:45,900 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 16:22:45,902 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 16:22:45,902 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 16:22:45,903 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 16:22:45,903 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 16:22:45,903 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 16:22:45,904 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 16:22:45,905 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 16:22:45,905 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 16:22:45,905 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 16:22:45,907 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 16:22:45,907 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 16:22:45,907 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 16:22:45,908 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 16:22:45,908 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 16:22:45,908 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 16:22:45,909 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 16:22:45,909 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 16:22:45,909 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 16:22:45,910 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 16:22:45,910 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 16:22:45,910 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 16:22:45,911 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 16:22:45,911 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 16:22:45,911 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 16:22:45,911 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-02 16:22:45,913 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 16:22:45,914 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-02 16:22:45,916 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-02 16:22:45,916 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-02 16:22:45,917 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-02 16:22:45,918 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-02 16:22:45,926 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-02 16:22:45,929 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-02 16:22:45,931 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-02 16:22:45,931 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-02 16:22:45,931 main DEBUG createAppenders(={Console}) 2024-12-02 16:22:45,932 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-02 16:22:45,933 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-02 16:22:45,933 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-02 16:22:45,934 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-02 16:22:45,934 main DEBUG OutputStream closed 2024-12-02 16:22:45,934 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-02 16:22:45,934 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-02 16:22:45,935 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-02 16:22:46,013 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-02 16:22:46,016 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-02 16:22:46,017 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-02 16:22:46,017 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-02 16:22:46,018 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-02 16:22:46,018 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-02 16:22:46,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-02 16:22:46,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-02 16:22:46,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-02 16:22:46,020 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-02 16:22:46,020 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-02 16:22:46,020 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-02 16:22:46,020 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-02 16:22:46,021 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-02 16:22:46,021 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-02 16:22:46,021 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-02 16:22:46,021 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-02 16:22:46,022 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-02 16:22:46,024 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02 16:22:46,025 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-02 16:22:46,025 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-02 16:22:46,026 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-02T16:22:46,239 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008 2024-12-02 16:22:46,241 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-02 16:22:46,242 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02T16:22:46,251 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-02T16:22:46,256 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-12-02T16:22:46,275 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-02T16:22:46,313 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-02T16:22:46,313 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-02T16:22:46,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T16:22:46,340 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38, deleteOnExit=true 2024-12-02T16:22:46,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T16:22:46,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/test.cache.data in system properties and HBase conf 2024-12-02T16:22:46,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T16:22:46,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/hadoop.log.dir in system properties and HBase conf 2024-12-02T16:22:46,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T16:22:46,343 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T16:22:46,343 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T16:22:46,417 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-02T16:22:46,499 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T16:22:46,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T16:22:46,504 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T16:22:46,504 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T16:22:46,505 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T16:22:46,505 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T16:22:46,506 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T16:22:46,506 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T16:22:46,507 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T16:22:46,507 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T16:22:46,507 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/nfs.dump.dir in system properties and HBase conf 2024-12-02T16:22:46,508 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/java.io.tmpdir in system properties and HBase conf 2024-12-02T16:22:46,508 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T16:22:46,508 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T16:22:46,508 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T16:22:47,390 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-02T16:22:47,455 INFO [Time-limited test {}] log.Log(170): Logging initialized @2298ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-02T16:22:47,519 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T16:22:47,582 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T16:22:47,603 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T16:22:47,604 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T16:22:47,605 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T16:22:47,617 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T16:22:47,620 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6bdd7391{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/hadoop.log.dir/,AVAILABLE} 2024-12-02T16:22:47,621 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4e578b56{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T16:22:47,771 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6878e65d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/java.io.tmpdir/jetty-localhost-41167-hadoop-hdfs-3_4_1-tests_jar-_-any-2019718272059465029/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T16:22:47,776 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2f30922e{HTTP/1.1, (http/1.1)}{localhost:41167} 2024-12-02T16:22:47,777 INFO [Time-limited test {}] server.Server(415): Started @2620ms 2024-12-02T16:22:48,239 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T16:22:48,246 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T16:22:48,247 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T16:22:48,247 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T16:22:48,248 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T16:22:48,248 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f494b95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/hadoop.log.dir/,AVAILABLE} 2024-12-02T16:22:48,249 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@63c29ae0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T16:22:48,345 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@388bacc3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/java.io.tmpdir/jetty-localhost-35267-hadoop-hdfs-3_4_1-tests_jar-_-any-7162755809516616742/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T16:22:48,346 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6f48fbbe{HTTP/1.1, (http/1.1)}{localhost:35267} 2024-12-02T16:22:48,346 INFO [Time-limited test {}] server.Server(415): Started @3190ms 2024-12-02T16:22:48,397 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T16:22:48,510 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T16:22:48,515 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T16:22:48,518 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T16:22:48,519 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T16:22:48,519 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T16:22:48,520 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3fe9d746{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/hadoop.log.dir/,AVAILABLE} 2024-12-02T16:22:48,521 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@540114bd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T16:22:48,647 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@565a2c18{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/java.io.tmpdir/jetty-localhost-46103-hadoop-hdfs-3_4_1-tests_jar-_-any-17129570961799934926/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T16:22:48,647 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@311fb016{HTTP/1.1, (http/1.1)}{localhost:46103} 2024-12-02T16:22:48,648 INFO [Time-limited test {}] server.Server(415): Started @3491ms 2024-12-02T16:22:48,650 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T16:22:48,684 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T16:22:48,689 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T16:22:48,691 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T16:22:48,691 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T16:22:48,691 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T16:22:48,692 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d7caafc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/hadoop.log.dir/,AVAILABLE} 2024-12-02T16:22:48,692 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@606c1a1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T16:22:48,788 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@b7f81b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/java.io.tmpdir/jetty-localhost-41701-hadoop-hdfs-3_4_1-tests_jar-_-any-8389565941234860052/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T16:22:48,788 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5d3aa315{HTTP/1.1, (http/1.1)}{localhost:41701} 2024-12-02T16:22:48,789 INFO [Time-limited test {}] server.Server(415): Started @3632ms 2024-12-02T16:22:48,790 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T16:22:49,604 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data1/current/BP-1520450069-172.17.0.2-1733156567009/current, will proceed with Du for space computation calculation, 2024-12-02T16:22:49,604 WARN [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data2/current/BP-1520450069-172.17.0.2-1733156567009/current, will proceed with Du for space computation calculation, 2024-12-02T16:22:49,630 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T16:22:49,672 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe1de610a22ebb29 with lease ID 0xf10224278278a23b: Processing first storage report for DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac from datanode DatanodeRegistration(127.0.0.1:33041, datanodeUuid=6d10331e-a0a5-46c6-bace-e0ef7d82b003, infoPort=33821, infoSecurePort=0, ipcPort=38325, storageInfo=lv=-57;cid=testClusterID;nsid=1188330875;c=1733156567009) 2024-12-02T16:22:49,674 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe1de610a22ebb29 with lease ID 0xf10224278278a23b: from storage DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac node DatanodeRegistration(127.0.0.1:33041, datanodeUuid=6d10331e-a0a5-46c6-bace-e0ef7d82b003, infoPort=33821, infoSecurePort=0, ipcPort=38325, storageInfo=lv=-57;cid=testClusterID;nsid=1188330875;c=1733156567009), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-02T16:22:49,675 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe1de610a22ebb29 with lease ID 0xf10224278278a23b: Processing first storage report for DS-eeb1ba11-3bc8-46c5-96fa-bbac23e3e624 from datanode DatanodeRegistration(127.0.0.1:33041, datanodeUuid=6d10331e-a0a5-46c6-bace-e0ef7d82b003, infoPort=33821, infoSecurePort=0, ipcPort=38325, storageInfo=lv=-57;cid=testClusterID;nsid=1188330875;c=1733156567009) 2024-12-02T16:22:49,675 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe1de610a22ebb29 with lease ID 0xf10224278278a23b: from storage DS-eeb1ba11-3bc8-46c5-96fa-bbac23e3e624 node DatanodeRegistration(127.0.0.1:33041, datanodeUuid=6d10331e-a0a5-46c6-bace-e0ef7d82b003, infoPort=33821, infoSecurePort=0, ipcPort=38325, storageInfo=lv=-57;cid=testClusterID;nsid=1188330875;c=1733156567009), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T16:22:49,675 WARN [Thread-131 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data3/current/BP-1520450069-172.17.0.2-1733156567009/current, will proceed with Du for space computation calculation, 2024-12-02T16:22:49,675 WARN [Thread-132 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data4/current/BP-1520450069-172.17.0.2-1733156567009/current, will proceed with Du for space computation calculation, 2024-12-02T16:22:49,705 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T16:22:49,710 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd5c74ab032eab81b with lease ID 0xf10224278278a23c: Processing first storage report for DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d from datanode DatanodeRegistration(127.0.0.1:38643, datanodeUuid=a5d22e8a-0746-4e0f-af23-884e49b6ed1e, infoPort=43815, infoSecurePort=0, ipcPort=38767, storageInfo=lv=-57;cid=testClusterID;nsid=1188330875;c=1733156567009) 2024-12-02T16:22:49,710 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd5c74ab032eab81b with lease ID 0xf10224278278a23c: from storage DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d node DatanodeRegistration(127.0.0.1:38643, datanodeUuid=a5d22e8a-0746-4e0f-af23-884e49b6ed1e, infoPort=43815, infoSecurePort=0, ipcPort=38767, storageInfo=lv=-57;cid=testClusterID;nsid=1188330875;c=1733156567009), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T16:22:49,710 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd5c74ab032eab81b with lease ID 0xf10224278278a23c: Processing first storage report for DS-85f24b05-b7f2-4ac0-a239-92b35f98e4a2 from datanode DatanodeRegistration(127.0.0.1:38643, datanodeUuid=a5d22e8a-0746-4e0f-af23-884e49b6ed1e, infoPort=43815, infoSecurePort=0, ipcPort=38767, storageInfo=lv=-57;cid=testClusterID;nsid=1188330875;c=1733156567009) 2024-12-02T16:22:49,711 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd5c74ab032eab81b with lease ID 0xf10224278278a23c: from storage DS-85f24b05-b7f2-4ac0-a239-92b35f98e4a2 node DatanodeRegistration(127.0.0.1:38643, datanodeUuid=a5d22e8a-0746-4e0f-af23-884e49b6ed1e, infoPort=43815, infoSecurePort=0, ipcPort=38767, storageInfo=lv=-57;cid=testClusterID;nsid=1188330875;c=1733156567009), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T16:22:49,790 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data6/current/BP-1520450069-172.17.0.2-1733156567009/current, will proceed with Du for space computation calculation, 2024-12-02T16:22:49,790 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data5/current/BP-1520450069-172.17.0.2-1733156567009/current, will proceed with Du for space computation calculation, 2024-12-02T16:22:49,815 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T16:22:49,820 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe22810545ef352c5 with lease ID 0xf10224278278a23d: Processing first storage report for DS-47476b29-867f-4268-be9e-3a4c1c5455c9 from datanode DatanodeRegistration(127.0.0.1:33271, datanodeUuid=e03a4d62-a0d6-4c80-951a-c1a27b850867, infoPort=35029, infoSecurePort=0, ipcPort=38415, storageInfo=lv=-57;cid=testClusterID;nsid=1188330875;c=1733156567009) 2024-12-02T16:22:49,820 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe22810545ef352c5 with lease ID 0xf10224278278a23d: from storage DS-47476b29-867f-4268-be9e-3a4c1c5455c9 node DatanodeRegistration(127.0.0.1:33271, datanodeUuid=e03a4d62-a0d6-4c80-951a-c1a27b850867, infoPort=35029, infoSecurePort=0, ipcPort=38415, storageInfo=lv=-57;cid=testClusterID;nsid=1188330875;c=1733156567009), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T16:22:49,821 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe22810545ef352c5 with lease ID 0xf10224278278a23d: Processing first storage report for DS-0d655588-ae90-4ceb-bd7a-c5d683f35ce2 from datanode DatanodeRegistration(127.0.0.1:33271, datanodeUuid=e03a4d62-a0d6-4c80-951a-c1a27b850867, infoPort=35029, infoSecurePort=0, ipcPort=38415, storageInfo=lv=-57;cid=testClusterID;nsid=1188330875;c=1733156567009) 2024-12-02T16:22:49,821 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe22810545ef352c5 with lease ID 0xf10224278278a23d: from storage DS-0d655588-ae90-4ceb-bd7a-c5d683f35ce2 node DatanodeRegistration(127.0.0.1:33271, datanodeUuid=e03a4d62-a0d6-4c80-951a-c1a27b850867, infoPort=35029, infoSecurePort=0, ipcPort=38415, storageInfo=lv=-57;cid=testClusterID;nsid=1188330875;c=1733156567009), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T16:22:49,885 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008 2024-12-02T16:22:49,970 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/zookeeper_0, clientPort=61407, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T16:22:49,980 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61407 2024-12-02T16:22:49,994 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T16:22:49,998 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T16:22:50,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741825_1001 (size=7) 2024-12-02T16:22:50,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741825_1001 (size=7) 2024-12-02T16:22:50,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741825_1001 (size=7) 2024-12-02T16:22:50,585 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b with version=8 2024-12-02T16:22:50,585 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/hbase-staging 2024-12-02T16:22:50,834 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e5d35c688fae:0 server-side Connection retries=45 2024-12-02T16:22:50,842 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T16:22:50,843 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T16:22:50,847 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T16:22:50,847 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T16:22:50,847 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T16:22:50,961 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T16:22:51,013 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-02T16:22:51,020 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-02T16:22:51,024 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T16:22:51,044 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 47591 (auto-detected) 2024-12-02T16:22:51,045 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-02T16:22:51,062 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38905 2024-12-02T16:22:51,080 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38905 connecting to ZooKeeper ensemble=127.0.0.1:61407 2024-12-02T16:22:51,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:389050x0, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T16:22:51,246 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38905-0x1009bb082170000 connected 2024-12-02T16:22:51,322 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T16:22:51,328 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T16:22:51,339 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T16:22:51,343 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b, hbase.cluster.distributed=false 2024-12-02T16:22:51,365 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T16:22:51,370 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38905 2024-12-02T16:22:51,371 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38905 2024-12-02T16:22:51,371 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38905 2024-12-02T16:22:51,372 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38905 2024-12-02T16:22:51,373 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38905 2024-12-02T16:22:51,475 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e5d35c688fae:0 server-side Connection retries=45 2024-12-02T16:22:51,477 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T16:22:51,477 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T16:22:51,477 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T16:22:51,477 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T16:22:51,477 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T16:22:51,480 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T16:22:51,482 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T16:22:51,483 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36399 2024-12-02T16:22:51,484 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36399 connecting to ZooKeeper ensemble=127.0.0.1:61407 2024-12-02T16:22:51,485 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T16:22:51,487 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T16:22:51,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:363990x0, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T16:22:51,502 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:363990x0, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T16:22:51,502 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36399-0x1009bb082170001 connected 2024-12-02T16:22:51,505 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T16:22:51,513 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T16:22:51,515 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36399-0x1009bb082170001, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T16:22:51,521 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36399-0x1009bb082170001, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T16:22:51,522 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36399 2024-12-02T16:22:51,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36399 2024-12-02T16:22:51,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36399 2024-12-02T16:22:51,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36399 2024-12-02T16:22:51,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36399 2024-12-02T16:22:51,540 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e5d35c688fae:0 server-side Connection retries=45 2024-12-02T16:22:51,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T16:22:51,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T16:22:51,541 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T16:22:51,541 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T16:22:51,541 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T16:22:51,541 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T16:22:51,541 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T16:22:51,542 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35297 2024-12-02T16:22:51,544 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35297 connecting to ZooKeeper ensemble=127.0.0.1:61407 2024-12-02T16:22:51,545 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T16:22:51,547 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T16:22:51,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:352970x0, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T16:22:51,562 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:352970x0, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T16:22:51,562 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35297-0x1009bb082170002 connected 2024-12-02T16:22:51,562 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T16:22:51,564 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T16:22:51,566 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T16:22:51,568 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T16:22:51,570 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35297 2024-12-02T16:22:51,570 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35297 2024-12-02T16:22:51,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35297 2024-12-02T16:22:51,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35297 2024-12-02T16:22:51,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35297 2024-12-02T16:22:51,587 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e5d35c688fae:0 server-side Connection retries=45 2024-12-02T16:22:51,587 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T16:22:51,587 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T16:22:51,587 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T16:22:51,587 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T16:22:51,588 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T16:22:51,588 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T16:22:51,588 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T16:22:51,589 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43901 2024-12-02T16:22:51,591 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43901 connecting to ZooKeeper ensemble=127.0.0.1:61407 2024-12-02T16:22:51,592 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T16:22:51,594 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T16:22:51,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:439010x0, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T16:22:51,604 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43901-0x1009bb082170003 connected 2024-12-02T16:22:51,604 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T16:22:51,605 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T16:22:51,606 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T16:22:51,608 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T16:22:51,610 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T16:22:51,613 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43901 2024-12-02T16:22:51,613 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43901 2024-12-02T16:22:51,614 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43901 2024-12-02T16:22:51,617 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43901 2024-12-02T16:22:51,617 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43901 2024-12-02T16:22:51,632 DEBUG [M:0;e5d35c688fae:38905 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e5d35c688fae:38905 2024-12-02T16:22:51,633 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e5d35c688fae,38905,1733156570684 2024-12-02T16:22:51,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T16:22:51,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T16:22:51,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36399-0x1009bb082170001, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T16:22:51,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T16:22:51,647 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e5d35c688fae,38905,1733156570684 2024-12-02T16:22:51,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36399-0x1009bb082170001, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T16:22:51,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T16:22:51,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:51,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36399-0x1009bb082170001, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:51,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T16:22:51,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:51,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:51,679 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T16:22:51,680 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e5d35c688fae,38905,1733156570684 from backup master directory 2024-12-02T16:22:51,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e5d35c688fae,38905,1733156570684 2024-12-02T16:22:51,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T16:22:51,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36399-0x1009bb082170001, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T16:22:51,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T16:22:51,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T16:22:51,687 WARN [master/e5d35c688fae:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T16:22:51,687 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e5d35c688fae,38905,1733156570684 2024-12-02T16:22:51,689 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-02T16:22:51,691 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-02T16:22:51,744 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/hbase.id] with ID: f5805b21-ffb2-47ac-9b04-2a283bcdbef5 2024-12-02T16:22:51,744 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/.tmp/hbase.id 2024-12-02T16:22:51,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741826_1002 (size=42) 2024-12-02T16:22:51,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741826_1002 (size=42) 2024-12-02T16:22:51,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741826_1002 (size=42) 2024-12-02T16:22:51,760 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/.tmp/hbase.id]:[hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/hbase.id] 2024-12-02T16:22:51,804 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T16:22:51,809 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T16:22:51,828 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-12-02T16:22:51,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:51,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:51,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:51,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36399-0x1009bb082170001, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:51,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741827_1003 (size=196) 2024-12-02T16:22:51,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741827_1003 (size=196) 2024-12-02T16:22:51,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741827_1003 (size=196) 2024-12-02T16:22:51,869 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T16:22:51,870 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T16:22:51,875 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:22:51,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741828_1004 (size=1189) 2024-12-02T16:22:51,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741828_1004 (size=1189) 2024-12-02T16:22:51,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741828_1004 (size=1189) 2024-12-02T16:22:51,924 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store 2024-12-02T16:22:51,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741829_1005 (size=34) 2024-12-02T16:22:51,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741829_1005 (size=34) 2024-12-02T16:22:51,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741829_1005 (size=34) 2024-12-02T16:22:51,949 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-02T16:22:51,953 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:22:51,954 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T16:22:51,955 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T16:22:51,955 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T16:22:51,957 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T16:22:51,957 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T16:22:51,957 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T16:22:51,958 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733156571954Disabling compacts and flushes for region at 1733156571954Disabling writes for close at 1733156571957 (+3 ms)Writing region close event to WAL at 1733156571957Closed at 1733156571957 2024-12-02T16:22:51,960 WARN [master/e5d35c688fae:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/.initializing 2024-12-02T16:22:51,961 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/WALs/e5d35c688fae,38905,1733156570684 2024-12-02T16:22:51,968 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T16:22:51,982 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e5d35c688fae%2C38905%2C1733156570684, suffix=, logDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/WALs/e5d35c688fae,38905,1733156570684, archiveDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/oldWALs, maxLogs=10 2024-12-02T16:22:52,011 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/WALs/e5d35c688fae,38905,1733156570684/e5d35c688fae%2C38905%2C1733156570684.1733156571987, exclude list is [], retry=0 2024-12-02T16:22:52,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:22:52,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:22:52,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:22:52,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:22:52,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-02T16:22:52,064 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-12-02T16:22:52,118 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/WALs/e5d35c688fae,38905,1733156570684/e5d35c688fae%2C38905%2C1733156570684.1733156571987 2024-12-02T16:22:52,119 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:35029:35029)] 2024-12-02T16:22:52,120 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:22:52,120 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:22:52,124 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T16:22:52,125 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T16:22:52,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T16:22:52,181 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T16:22:52,184 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:52,187 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T16:22:52,187 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T16:22:52,191 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T16:22:52,191 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:52,192 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:22:52,192 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T16:22:52,195 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T16:22:52,195 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:52,196 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:22:52,196 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T16:22:52,199 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T16:22:52,199 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:52,200 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:22:52,200 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T16:22:52,204 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T16:22:52,205 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T16:22:52,210 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T16:22:52,211 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T16:22:52,214 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T16:22:52,217 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T16:22:52,221 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T16:22:52,223 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69652862, jitterRate=0.03790852427482605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T16:22:52,228 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733156572136Initializing all the Stores at 1733156572138 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733156572139 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156572139Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156572140 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156572140Cleaning up temporary data from old regions at 1733156572211 (+71 ms)Region opened successfully at 1733156572228 (+17 ms) 2024-12-02T16:22:52,230 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T16:22:52,261 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663f6ef9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e5d35c688fae/172.17.0.2:0 2024-12-02T16:22:52,289 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T16:22:52,298 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-02T16:22:52,299 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-02T16:22:52,301 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T16:22:52,302 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-02T16:22:52,307 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-02T16:22:52,307 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T16:22:52,331 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T16:22:52,339 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T16:22:52,376 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T16:22:52,379 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T16:22:52,381 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T16:22:52,392 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T16:22:52,394 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T16:22:52,397 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T16:22:52,402 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T16:22:52,404 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T16:22:52,411 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T16:22:52,431 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T16:22:52,442 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T16:22:52,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T16:22:52,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T16:22:52,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36399-0x1009bb082170001, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T16:22:52,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T16:22:52,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:52,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36399-0x1009bb082170001, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:52,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:52,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:52,458 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e5d35c688fae,38905,1733156570684, sessionid=0x1009bb082170000, setting cluster-up flag (Was=false) 2024-12-02T16:22:52,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:52,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:52,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:52,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36399-0x1009bb082170001, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:52,512 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T16:22:52,516 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e5d35c688fae,38905,1733156570684 2024-12-02T16:22:52,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:52,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36399-0x1009bb082170001, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:52,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:52,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:52,561 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T16:22:52,564 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e5d35c688fae,38905,1733156570684 2024-12-02T16:22:52,573 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T16:22:52,623 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.HRegionServer(746): ClusterId : f5805b21-ffb2-47ac-9b04-2a283bcdbef5 2024-12-02T16:22:52,623 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.HRegionServer(746): ClusterId : f5805b21-ffb2-47ac-9b04-2a283bcdbef5 2024-12-02T16:22:52,623 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.HRegionServer(746): ClusterId : f5805b21-ffb2-47ac-9b04-2a283bcdbef5 2024-12-02T16:22:52,625 DEBUG [RS:1;e5d35c688fae:35297 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T16:22:52,625 DEBUG [RS:2;e5d35c688fae:43901 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T16:22:52,625 DEBUG [RS:0;e5d35c688fae:36399 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T16:22:52,638 DEBUG [RS:0;e5d35c688fae:36399 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T16:22:52,638 DEBUG [RS:1;e5d35c688fae:35297 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T16:22:52,638 DEBUG [RS:2;e5d35c688fae:43901 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T16:22:52,639 DEBUG [RS:0;e5d35c688fae:36399 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T16:22:52,639 DEBUG [RS:1;e5d35c688fae:35297 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T16:22:52,639 DEBUG [RS:2;e5d35c688fae:43901 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T16:22:52,646 DEBUG [RS:0;e5d35c688fae:36399 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T16:22:52,646 DEBUG [RS:1;e5d35c688fae:35297 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T16:22:52,646 DEBUG [RS:2;e5d35c688fae:43901 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T16:22:52,646 DEBUG [RS:2;e5d35c688fae:43901 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50202ec5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e5d35c688fae/172.17.0.2:0 2024-12-02T16:22:52,646 DEBUG [RS:0;e5d35c688fae:36399 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2092c37f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e5d35c688fae/172.17.0.2:0 2024-12-02T16:22:52,646 DEBUG [RS:1;e5d35c688fae:35297 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20af5d12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e5d35c688fae/172.17.0.2:0 2024-12-02T16:22:52,646 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T16:22:52,656 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T16:22:52,659 DEBUG [RS:0;e5d35c688fae:36399 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e5d35c688fae:36399 2024-12-02T16:22:52,662 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T16:22:52,662 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T16:22:52,662 DEBUG [RS:0;e5d35c688fae:36399 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T16:22:52,663 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T16:22:52,665 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.HRegionServer(2659): reportForDuty to master=e5d35c688fae,38905,1733156570684 with port=36399, startcode=1733156571440 2024-12-02T16:22:52,665 DEBUG [RS:1;e5d35c688fae:35297 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;e5d35c688fae:35297 2024-12-02T16:22:52,665 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T16:22:52,665 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T16:22:52,666 DEBUG [RS:1;e5d35c688fae:35297 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T16:22:52,666 DEBUG [RS:2;e5d35c688fae:43901 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;e5d35c688fae:43901 2024-12-02T16:22:52,666 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T16:22:52,666 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T16:22:52,666 DEBUG [RS:2;e5d35c688fae:43901 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T16:22:52,667 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.HRegionServer(2659): reportForDuty to master=e5d35c688fae,38905,1733156570684 with port=35297, startcode=1733156571540 2024-12-02T16:22:52,667 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.HRegionServer(2659): reportForDuty to master=e5d35c688fae,38905,1733156570684 with port=43901, startcode=1733156571586 2024-12-02T16:22:52,670 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e5d35c688fae,38905,1733156570684 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T16:22:52,676 DEBUG [RS:0;e5d35c688fae:36399 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T16:22:52,676 DEBUG [RS:1;e5d35c688fae:35297 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T16:22:52,676 DEBUG [RS:2;e5d35c688fae:43901 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T16:22:52,678 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e5d35c688fae:0, corePoolSize=5, maxPoolSize=5 2024-12-02T16:22:52,678 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e5d35c688fae:0, corePoolSize=5, maxPoolSize=5 2024-12-02T16:22:52,678 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e5d35c688fae:0, corePoolSize=5, maxPoolSize=5 2024-12-02T16:22:52,678 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e5d35c688fae:0, corePoolSize=5, maxPoolSize=5 2024-12-02T16:22:52,679 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e5d35c688fae:0, corePoolSize=10, maxPoolSize=10 2024-12-02T16:22:52,679 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,679 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e5d35c688fae:0, corePoolSize=2, maxPoolSize=2 2024-12-02T16:22:52,679 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,686 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T16:22:52,687 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T16:22:52,687 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733156602685 2024-12-02T16:22:52,689 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T16:22:52,690 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T16:22:52,694 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T16:22:52,694 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:52,695 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T16:22:52,695 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T16:22:52,696 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T16:22:52,696 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T16:22:52,697 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,701 WARN [IPC Server handler 1 on default port 33443 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-02T16:22:52,701 WARN [IPC Server handler 1 on default port 33443 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-02T16:22:52,701 WARN [IPC Server handler 1 on default port 33443 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-02T16:22:52,704 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T16:22:52,706 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T16:22:52,706 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T16:22:52,709 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T16:22:52,709 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T16:22:52,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741831_1007 (size=1321) 2024-12-02T16:22:52,712 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e5d35c688fae:0:becomeActiveMaster-HFileCleaner.large.0-1733156572711,5,FailOnTimeoutGroup] 2024-12-02T16:22:52,714 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T16:22:52,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741831_1007 (size=1321) 2024-12-02T16:22:52,714 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b 2024-12-02T16:22:52,718 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e5d35c688fae:0:becomeActiveMaster-HFileCleaner.small.0-1733156572712,5,FailOnTimeoutGroup] 2024-12-02T16:22:52,718 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,718 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T16:22:52,723 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,724 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,724 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55453, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T16:22:52,724 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44357, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T16:22:52,724 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56703, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T16:22:52,758 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38905 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e5d35c688fae,35297,1733156571540 2024-12-02T16:22:52,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741832_1008 (size=32) 2024-12-02T16:22:52,761 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38905 {}] master.ServerManager(517): Registering regionserver=e5d35c688fae,35297,1733156571540 2024-12-02T16:22:52,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741832_1008 (size=32) 2024-12-02T16:22:52,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741832_1008 (size=32) 2024-12-02T16:22:52,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:22:52,767 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T16:22:52,770 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T16:22:52,770 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:52,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T16:22:52,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T16:22:52,774 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T16:22:52,774 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:52,775 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38905 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e5d35c688fae,43901,1733156571586 2024-12-02T16:22:52,775 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T16:22:52,776 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38905 {}] master.ServerManager(517): Registering regionserver=e5d35c688fae,43901,1733156571586 2024-12-02T16:22:52,776 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T16:22:52,779 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T16:22:52,779 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:52,779 DEBUG [RS:1;e5d35c688fae:35297 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b 2024-12-02T16:22:52,780 DEBUG [RS:1;e5d35c688fae:35297 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33443 2024-12-02T16:22:52,780 DEBUG [RS:1;e5d35c688fae:35297 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T16:22:52,781 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T16:22:52,781 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T16:22:52,781 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38905 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e5d35c688fae,36399,1733156571440 2024-12-02T16:22:52,781 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38905 {}] master.ServerManager(517): Registering regionserver=e5d35c688fae,36399,1733156571440 2024-12-02T16:22:52,781 DEBUG [RS:2;e5d35c688fae:43901 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b 2024-12-02T16:22:52,782 DEBUG [RS:2;e5d35c688fae:43901 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33443 2024-12-02T16:22:52,782 DEBUG [RS:2;e5d35c688fae:43901 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T16:22:52,784 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T16:22:52,785 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:52,785 DEBUG [RS:0;e5d35c688fae:36399 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b 2024-12-02T16:22:52,785 DEBUG [RS:0;e5d35c688fae:36399 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33443 2024-12-02T16:22:52,785 DEBUG [RS:0;e5d35c688fae:36399 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T16:22:52,786 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T16:22:52,786 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T16:22:52,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740 2024-12-02T16:22:52,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740 2024-12-02T16:22:52,791 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T16:22:52,792 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T16:22:52,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T16:22:52,793 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T16:22:52,796 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T16:22:52,801 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T16:22:52,803 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63431207, jitterRate=-0.05480135977268219}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T16:22:52,806 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733156572764Initializing all the Stores at 1733156572766 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733156572767 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733156572767Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156572767Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733156572767Cleaning up temporary data from old regions at 1733156572792 (+25 ms)Region opened successfully at 1733156572805 (+13 ms) 2024-12-02T16:22:52,806 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T16:22:52,806 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T16:22:52,806 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T16:22:52,806 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T16:22:52,806 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T16:22:52,808 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T16:22:52,808 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733156572806Disabling compacts and flushes for region at 1733156572806Disabling writes for close at 1733156572806Writing region close event to WAL at 1733156572808 (+2 ms)Closed at 1733156572808 2024-12-02T16:22:52,812 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T16:22:52,812 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T16:22:52,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T16:22:52,824 DEBUG [RS:1;e5d35c688fae:35297 {}] zookeeper.ZKUtil(111): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e5d35c688fae,35297,1733156571540 2024-12-02T16:22:52,825 DEBUG [RS:0;e5d35c688fae:36399 {}] zookeeper.ZKUtil(111): regionserver:36399-0x1009bb082170001, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e5d35c688fae,36399,1733156571440 2024-12-02T16:22:52,825 DEBUG [RS:2;e5d35c688fae:43901 {}] zookeeper.ZKUtil(111): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e5d35c688fae,43901,1733156571586 2024-12-02T16:22:52,825 WARN [RS:1;e5d35c688fae:35297 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T16:22:52,825 WARN [RS:0;e5d35c688fae:36399 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T16:22:52,825 WARN [RS:2;e5d35c688fae:43901 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T16:22:52,825 INFO [RS:1;e5d35c688fae:35297 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:22:52,825 INFO [RS:2;e5d35c688fae:43901 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:22:52,825 INFO [RS:0;e5d35c688fae:36399 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:22:52,825 DEBUG [RS:1;e5d35c688fae:35297 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,35297,1733156571540 2024-12-02T16:22:52,825 DEBUG [RS:2;e5d35c688fae:43901 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,43901,1733156571586 2024-12-02T16:22:52,825 DEBUG [RS:0;e5d35c688fae:36399 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440 2024-12-02T16:22:52,827 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T16:22:52,827 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e5d35c688fae,35297,1733156571540] 2024-12-02T16:22:52,827 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e5d35c688fae,43901,1733156571586] 2024-12-02T16:22:52,827 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e5d35c688fae,36399,1733156571440] 2024-12-02T16:22:52,831 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T16:22:52,850 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T16:22:52,850 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T16:22:52,850 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T16:22:52,862 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T16:22:52,862 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T16:22:52,862 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T16:22:52,867 INFO [RS:1;e5d35c688fae:35297 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T16:22:52,867 INFO [RS:2;e5d35c688fae:43901 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T16:22:52,867 INFO [RS:0;e5d35c688fae:36399 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T16:22:52,867 INFO [RS:1;e5d35c688fae:35297 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,867 INFO [RS:0;e5d35c688fae:36399 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,867 INFO [RS:2;e5d35c688fae:43901 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,868 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T16:22:52,869 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T16:22:52,869 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T16:22:52,874 INFO [RS:1;e5d35c688fae:35297 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T16:22:52,874 INFO [RS:0;e5d35c688fae:36399 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T16:22:52,874 INFO [RS:2;e5d35c688fae:43901 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T16:22:52,875 INFO [RS:1;e5d35c688fae:35297 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,875 INFO [RS:2;e5d35c688fae:43901 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,875 INFO [RS:0;e5d35c688fae:36399 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,876 DEBUG [RS:1;e5d35c688fae:35297 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,876 DEBUG [RS:0;e5d35c688fae:36399 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,876 DEBUG [RS:2;e5d35c688fae:43901 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,876 DEBUG [RS:2;e5d35c688fae:43901 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,876 DEBUG [RS:0;e5d35c688fae:36399 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,876 DEBUG [RS:1;e5d35c688fae:35297 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,876 DEBUG [RS:0;e5d35c688fae:36399 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,876 DEBUG [RS:2;e5d35c688fae:43901 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,876 DEBUG [RS:1;e5d35c688fae:35297 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,876 DEBUG [RS:0;e5d35c688fae:36399 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,876 DEBUG [RS:1;e5d35c688fae:35297 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,876 DEBUG [RS:2;e5d35c688fae:43901 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,876 DEBUG [RS:0;e5d35c688fae:36399 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,876 DEBUG [RS:2;e5d35c688fae:43901 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,876 DEBUG [RS:1;e5d35c688fae:35297 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:0;e5d35c688fae:36399 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0, corePoolSize=2, maxPoolSize=2 2024-12-02T16:22:52,877 DEBUG [RS:2;e5d35c688fae:43901 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0, corePoolSize=2, maxPoolSize=2 2024-12-02T16:22:52,877 DEBUG [RS:1;e5d35c688fae:35297 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0, corePoolSize=2, maxPoolSize=2 2024-12-02T16:22:52,877 DEBUG [RS:0;e5d35c688fae:36399 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:2;e5d35c688fae:43901 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:1;e5d35c688fae:35297 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:0;e5d35c688fae:36399 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:2;e5d35c688fae:43901 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:1;e5d35c688fae:35297 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:2;e5d35c688fae:43901 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:0;e5d35c688fae:36399 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:1;e5d35c688fae:35297 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:2;e5d35c688fae:43901 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:0;e5d35c688fae:36399 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:1;e5d35c688fae:35297 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:0;e5d35c688fae:36399 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:2;e5d35c688fae:43901 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:1;e5d35c688fae:35297 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:2;e5d35c688fae:43901 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:0;e5d35c688fae:36399 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,877 DEBUG [RS:1;e5d35c688fae:35297 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e5d35c688fae:0, corePoolSize=1, maxPoolSize=1 2024-12-02T16:22:52,878 DEBUG [RS:2;e5d35c688fae:43901 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e5d35c688fae:0, corePoolSize=3, maxPoolSize=3 2024-12-02T16:22:52,878 DEBUG [RS:0;e5d35c688fae:36399 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e5d35c688fae:0, corePoolSize=3, maxPoolSize=3 2024-12-02T16:22:52,878 DEBUG [RS:1;e5d35c688fae:35297 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e5d35c688fae:0, corePoolSize=3, maxPoolSize=3 2024-12-02T16:22:52,878 DEBUG [RS:0;e5d35c688fae:36399 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e5d35c688fae:0, corePoolSize=3, maxPoolSize=3 2024-12-02T16:22:52,878 DEBUG [RS:2;e5d35c688fae:43901 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e5d35c688fae:0, corePoolSize=3, maxPoolSize=3 2024-12-02T16:22:52,878 DEBUG [RS:1;e5d35c688fae:35297 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e5d35c688fae:0, corePoolSize=3, maxPoolSize=3 2024-12-02T16:22:52,881 INFO [RS:0;e5d35c688fae:36399 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,881 INFO [RS:2;e5d35c688fae:43901 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,881 INFO [RS:1;e5d35c688fae:35297 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,881 INFO [RS:0;e5d35c688fae:36399 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,881 INFO [RS:2;e5d35c688fae:43901 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,881 INFO [RS:1;e5d35c688fae:35297 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,881 INFO [RS:2;e5d35c688fae:43901 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,881 INFO [RS:1;e5d35c688fae:35297 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,881 INFO [RS:2;e5d35c688fae:43901 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,881 INFO [RS:1;e5d35c688fae:35297 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,881 INFO [RS:0;e5d35c688fae:36399 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,881 INFO [RS:2;e5d35c688fae:43901 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,881 INFO [RS:0;e5d35c688fae:36399 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,881 INFO [RS:1;e5d35c688fae:35297 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,881 INFO [RS:2;e5d35c688fae:43901 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5d35c688fae,43901,1733156571586-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T16:22:52,881 INFO [RS:0;e5d35c688fae:36399 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,881 INFO [RS:1;e5d35c688fae:35297 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5d35c688fae,35297,1733156571540-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T16:22:52,881 INFO [RS:0;e5d35c688fae:36399 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5d35c688fae,36399,1733156571440-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T16:22:52,899 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T16:22:52,901 INFO [RS:2;e5d35c688fae:43901 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5d35c688fae,43901,1733156571586-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,901 INFO [RS:2;e5d35c688fae:43901 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,902 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.Replication(171): e5d35c688fae,43901,1733156571586 started 2024-12-02T16:22:52,902 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T16:22:52,902 INFO [RS:1;e5d35c688fae:35297 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5d35c688fae,35297,1733156571540-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,902 INFO [RS:1;e5d35c688fae:35297 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,902 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.Replication(171): e5d35c688fae,35297,1733156571540 started 2024-12-02T16:22:52,905 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T16:22:52,905 INFO [RS:0;e5d35c688fae:36399 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5d35c688fae,36399,1733156571440-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,905 INFO [RS:0;e5d35c688fae:36399 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,906 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.Replication(171): e5d35c688fae,36399,1733156571440 started 2024-12-02T16:22:52,918 INFO [RS:2;e5d35c688fae:43901 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,918 INFO [RS:1;e5d35c688fae:35297 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,918 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.HRegionServer(1482): Serving as e5d35c688fae,35297,1733156571540, RpcServer on e5d35c688fae/172.17.0.2:35297, sessionid=0x1009bb082170002 2024-12-02T16:22:52,918 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.HRegionServer(1482): Serving as e5d35c688fae,43901,1733156571586, RpcServer on e5d35c688fae/172.17.0.2:43901, sessionid=0x1009bb082170003 2024-12-02T16:22:52,919 DEBUG [RS:1;e5d35c688fae:35297 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T16:22:52,919 DEBUG [RS:2;e5d35c688fae:43901 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T16:22:52,919 DEBUG [RS:1;e5d35c688fae:35297 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e5d35c688fae,35297,1733156571540 2024-12-02T16:22:52,919 DEBUG [RS:2;e5d35c688fae:43901 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e5d35c688fae,43901,1733156571586 2024-12-02T16:22:52,919 DEBUG [RS:2;e5d35c688fae:43901 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5d35c688fae,43901,1733156571586' 2024-12-02T16:22:52,919 DEBUG [RS:1;e5d35c688fae:35297 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5d35c688fae,35297,1733156571540' 2024-12-02T16:22:52,919 DEBUG [RS:2;e5d35c688fae:43901 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T16:22:52,919 DEBUG [RS:1;e5d35c688fae:35297 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T16:22:52,920 DEBUG [RS:2;e5d35c688fae:43901 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T16:22:52,920 DEBUG [RS:1;e5d35c688fae:35297 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T16:22:52,920 INFO [RS:0;e5d35c688fae:36399 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:52,921 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.HRegionServer(1482): Serving as e5d35c688fae,36399,1733156571440, RpcServer on e5d35c688fae/172.17.0.2:36399, sessionid=0x1009bb082170001 2024-12-02T16:22:52,921 DEBUG [RS:2;e5d35c688fae:43901 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T16:22:52,921 DEBUG [RS:1;e5d35c688fae:35297 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T16:22:52,921 DEBUG [RS:2;e5d35c688fae:43901 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T16:22:52,921 DEBUG [RS:1;e5d35c688fae:35297 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T16:22:52,921 DEBUG [RS:0;e5d35c688fae:36399 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T16:22:52,921 DEBUG [RS:1;e5d35c688fae:35297 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e5d35c688fae,35297,1733156571540 2024-12-02T16:22:52,921 DEBUG [RS:2;e5d35c688fae:43901 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e5d35c688fae,43901,1733156571586 2024-12-02T16:22:52,921 DEBUG [RS:1;e5d35c688fae:35297 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5d35c688fae,35297,1733156571540' 2024-12-02T16:22:52,921 DEBUG [RS:0;e5d35c688fae:36399 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e5d35c688fae,36399,1733156571440 2024-12-02T16:22:52,921 DEBUG [RS:2;e5d35c688fae:43901 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5d35c688fae,43901,1733156571586' 2024-12-02T16:22:52,921 DEBUG [RS:1;e5d35c688fae:35297 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T16:22:52,921 DEBUG [RS:2;e5d35c688fae:43901 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T16:22:52,921 DEBUG [RS:0;e5d35c688fae:36399 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5d35c688fae,36399,1733156571440' 2024-12-02T16:22:52,921 DEBUG [RS:0;e5d35c688fae:36399 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T16:22:52,921 DEBUG [RS:2;e5d35c688fae:43901 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T16:22:52,922 DEBUG [RS:1;e5d35c688fae:35297 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T16:22:52,922 DEBUG [RS:0;e5d35c688fae:36399 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T16:22:52,922 DEBUG [RS:2;e5d35c688fae:43901 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T16:22:52,922 DEBUG [RS:1;e5d35c688fae:35297 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T16:22:52,922 INFO [RS:2;e5d35c688fae:43901 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T16:22:52,922 INFO [RS:1;e5d35c688fae:35297 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T16:22:52,922 DEBUG [RS:0;e5d35c688fae:36399 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T16:22:52,922 INFO [RS:2;e5d35c688fae:43901 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T16:22:52,922 INFO [RS:1;e5d35c688fae:35297 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T16:22:52,922 DEBUG [RS:0;e5d35c688fae:36399 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T16:22:52,922 DEBUG [RS:0;e5d35c688fae:36399 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e5d35c688fae,36399,1733156571440 2024-12-02T16:22:52,922 DEBUG [RS:0;e5d35c688fae:36399 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5d35c688fae,36399,1733156571440' 2024-12-02T16:22:52,922 DEBUG [RS:0;e5d35c688fae:36399 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T16:22:52,923 DEBUG [RS:0;e5d35c688fae:36399 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T16:22:52,923 DEBUG [RS:0;e5d35c688fae:36399 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T16:22:52,923 INFO [RS:0;e5d35c688fae:36399 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T16:22:52,923 INFO [RS:0;e5d35c688fae:36399 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T16:22:52,982 WARN [e5d35c688fae:38905 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-02T16:22:53,034 INFO [RS:1;e5d35c688fae:35297 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T16:22:53,034 INFO [RS:0;e5d35c688fae:36399 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T16:22:53,034 INFO [RS:2;e5d35c688fae:43901 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T16:22:53,037 INFO [RS:0;e5d35c688fae:36399 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e5d35c688fae%2C36399%2C1733156571440, suffix=, logDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440, archiveDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/oldWALs, maxLogs=32 2024-12-02T16:22:53,037 INFO [RS:2;e5d35c688fae:43901 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e5d35c688fae%2C43901%2C1733156571586, suffix=, logDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,43901,1733156571586, archiveDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/oldWALs, maxLogs=32 2024-12-02T16:22:53,037 INFO [RS:1;e5d35c688fae:35297 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e5d35c688fae%2C35297%2C1733156571540, suffix=, logDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,35297,1733156571540, archiveDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/oldWALs, maxLogs=32 2024-12-02T16:22:53,054 DEBUG [RS:2;e5d35c688fae:43901 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,43901,1733156571586/e5d35c688fae%2C43901%2C1733156571586.1733156573042, exclude list is [], retry=0 2024-12-02T16:22:53,054 DEBUG [RS:1;e5d35c688fae:35297 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,35297,1733156571540/e5d35c688fae%2C35297%2C1733156571540.1733156573042, exclude list is [], retry=0 2024-12-02T16:22:53,055 DEBUG [RS:0;e5d35c688fae:36399 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440/e5d35c688fae%2C36399%2C1733156571440.1733156573042, exclude list is [], retry=0 2024-12-02T16:22:53,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:22:53,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:22:53,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:22:53,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:22:53,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:22:53,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:22:53,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:22:53,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:22:53,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:22:53,100 INFO [RS:0;e5d35c688fae:36399 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440/e5d35c688fae%2C36399%2C1733156571440.1733156573042 2024-12-02T16:22:53,101 DEBUG [RS:0;e5d35c688fae:36399 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:33821:33821)] 2024-12-02T16:22:53,106 INFO [RS:1;e5d35c688fae:35297 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,35297,1733156571540/e5d35c688fae%2C35297%2C1733156571540.1733156573042 2024-12-02T16:22:53,106 INFO [RS:2;e5d35c688fae:43901 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,43901,1733156571586/e5d35c688fae%2C43901%2C1733156571586.1733156573042 2024-12-02T16:22:53,107 DEBUG [RS:1;e5d35c688fae:35297 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:33821:33821)] 2024-12-02T16:22:53,107 DEBUG [RS:2;e5d35c688fae:43901 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:33821:33821)] 2024-12-02T16:22:53,235 DEBUG [e5d35c688fae:38905 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-02T16:22:53,242 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(204): Hosts are {e5d35c688fae=0} racks are {/default-rack=0} 2024-12-02T16:22:53,248 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T16:22:53,248 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T16:22:53,248 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-02T16:22:53,249 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T16:22:53,249 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T16:22:53,249 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-02T16:22:53,249 INFO [e5d35c688fae:38905 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T16:22:53,249 INFO [e5d35c688fae:38905 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T16:22:53,249 INFO [e5d35c688fae:38905 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-02T16:22:53,249 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T16:22:53,255 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e5d35c688fae,36399,1733156571440 2024-12-02T16:22:53,260 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e5d35c688fae,36399,1733156571440, state=OPENING 2024-12-02T16:22:53,292 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T16:22:53,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:53,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:53,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:53,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36399-0x1009bb082170001, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:22:53,303 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T16:22:53,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T16:22:53,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T16:22:53,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T16:22:53,306 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T16:22:53,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e5d35c688fae,36399,1733156571440}] 2024-12-02T16:22:53,487 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T16:22:53,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46025, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T16:22:53,503 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T16:22:53,503 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:22:53,504 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-02T16:22:53,506 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e5d35c688fae%2C36399%2C1733156571440.meta, suffix=.meta, logDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440, archiveDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/oldWALs, maxLogs=32 2024-12-02T16:22:53,521 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440/e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta, exclude list is [], retry=0 2024-12-02T16:22:53,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:22:53,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:22:53,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:22:53,555 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440/e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta 2024-12-02T16:22:53,556 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:43815:43815)] 2024-12-02T16:22:53,556 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:22:53,558 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T16:22:53,561 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T16:22:53,567 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T16:22:53,572 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T16:22:53,573 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:22:53,573 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T16:22:53,573 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T16:22:53,577 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T16:22:53,578 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T16:22:53,578 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:53,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T16:22:53,580 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T16:22:53,581 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T16:22:53,581 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:53,582 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T16:22:53,582 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T16:22:53,583 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T16:22:53,583 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:53,584 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T16:22:53,584 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T16:22:53,586 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T16:22:53,586 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:53,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T16:22:53,587 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T16:22:53,588 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740 2024-12-02T16:22:53,592 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740 2024-12-02T16:22:53,594 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T16:22:53,595 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T16:22:53,596 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T16:22:53,599 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T16:22:53,601 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59286664, jitterRate=-0.11655986309051514}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T16:22:53,601 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T16:22:53,603 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733156573574Writing region info on filesystem at 1733156573574Initializing all the Stores at 1733156573576 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733156573576Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733156573577 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156573577Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733156573577Cleaning up temporary data from old regions at 1733156573595 (+18 ms)Running coprocessor post-open hooks at 1733156573601 (+6 ms)Region opened successfully at 1733156573603 (+2 ms) 2024-12-02T16:22:53,611 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733156573479 2024-12-02T16:22:53,623 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T16:22:53,623 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T16:22:53,625 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e5d35c688fae,36399,1733156571440 2024-12-02T16:22:53,628 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e5d35c688fae,36399,1733156571440, state=OPEN 2024-12-02T16:22:53,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T16:22:53,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36399-0x1009bb082170001, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T16:22:53,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T16:22:53,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T16:22:53,676 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T16:22:53,676 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T16:22:53,676 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T16:22:53,676 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T16:22:53,676 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e5d35c688fae,36399,1733156571440 2024-12-02T16:22:53,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T16:22:53,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e5d35c688fae,36399,1733156571440 in 368 msec 2024-12-02T16:22:53,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T16:22:53,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 865 msec 2024-12-02T16:22:53,693 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T16:22:53,693 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T16:22:53,734 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T16:22:53,735 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1] 2024-12-02T16:22:53,768 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T16:22:53,771 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55097, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T16:22:53,791 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1840 sec 2024-12-02T16:22:53,792 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733156573792, completionTime=-1 2024-12-02T16:22:53,794 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-02T16:22:53,794 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T16:22:53,823 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-02T16:22:53,823 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733156633823 2024-12-02T16:22:53,823 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733156693823 2024-12-02T16:22:53,823 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-12-02T16:22:53,825 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-02T16:22:53,831 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5d35c688fae,38905,1733156570684-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:53,832 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5d35c688fae,38905,1733156570684-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:53,832 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5d35c688fae,38905,1733156570684-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:53,833 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e5d35c688fae:38905, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:53,834 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:53,834 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:53,840 DEBUG [master/e5d35c688fae:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T16:22:53,866 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.179sec 2024-12-02T16:22:53,868 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T16:22:53,869 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T16:22:53,870 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T16:22:53,870 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T16:22:53,870 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T16:22:53,871 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5d35c688fae,38905,1733156570684-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T16:22:53,871 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5d35c688fae,38905,1733156570684-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T16:22:53,875 DEBUG [master/e5d35c688fae:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T16:22:53,876 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T16:22:53,877 INFO [master/e5d35c688fae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5d35c688fae,38905,1733156570684-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T16:22:53,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7698bd87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T16:22:53,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5d35c688fae,38905,-1 for getting cluster id 2024-12-02T16:22:53,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T16:22:53,950 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5805b21-ffb2-47ac-9b04-2a283bcdbef5' 2024-12-02T16:22:53,952 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T16:22:53,953 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5805b21-ffb2-47ac-9b04-2a283bcdbef5" 2024-12-02T16:22:53,953 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14dccdd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T16:22:53,953 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5d35c688fae,38905,-1] 2024-12-02T16:22:53,956 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T16:22:53,958 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T16:22:53,959 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T16:22:53,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64da1502, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T16:22:53,962 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T16:22:53,969 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1] 2024-12-02T16:22:53,970 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T16:22:53,972 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T16:22:53,992 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e5d35c688fae,38905,1733156570684 2024-12-02T16:22:53,992 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:33443/hbase 2024-12-02T16:22:54,009 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=362, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=275, ProcessCount=11, AvailableMemoryMB=6321 2024-12-02T16:22:54,028 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:22:54,031 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:22:54,032 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T16:22:54,037 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-27690357, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/hregion-27690357, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:22:54,051 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-27690357/hregion-27690357.1733156574038, exclude list is [], retry=0 2024-12-02T16:22:54,055 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:22:54,055 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:22:54,057 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:22:54,085 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-27690357/hregion-27690357.1733156574038 2024-12-02T16:22:54,086 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:43815:43815)] 2024-12-02T16:22:54,086 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 96053ec1b36370e6b1bdd61b2f5d49cb, NAME => 'testReplayEditsWrittenIntoWAL,,1733156574028.96053ec1b36370e6b1bdd61b2f5d49cb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33443/hbase 2024-12-02T16:22:54,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741838_1014 (size=64) 2024-12-02T16:22:54,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741838_1014 (size=64) 2024-12-02T16:22:54,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741838_1014 (size=64) 2024-12-02T16:22:54,100 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733156574028.96053ec1b36370e6b1bdd61b2f5d49cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:22:54,102 INFO [StoreOpener-96053ec1b36370e6b1bdd61b2f5d49cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 96053ec1b36370e6b1bdd61b2f5d49cb 2024-12-02T16:22:54,105 INFO [StoreOpener-96053ec1b36370e6b1bdd61b2f5d49cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96053ec1b36370e6b1bdd61b2f5d49cb columnFamilyName a 2024-12-02T16:22:54,105 DEBUG [StoreOpener-96053ec1b36370e6b1bdd61b2f5d49cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:54,106 INFO [StoreOpener-96053ec1b36370e6b1bdd61b2f5d49cb-1 {}] regionserver.HStore(327): Store=96053ec1b36370e6b1bdd61b2f5d49cb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:22:54,106 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 96053ec1b36370e6b1bdd61b2f5d49cb 2024-12-02T16:22:54,108 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb 2024-12-02T16:22:54,109 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb 2024-12-02T16:22:54,109 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 96053ec1b36370e6b1bdd61b2f5d49cb 2024-12-02T16:22:54,109 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 96053ec1b36370e6b1bdd61b2f5d49cb 2024-12-02T16:22:54,112 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 96053ec1b36370e6b1bdd61b2f5d49cb 2024-12-02T16:22:54,115 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T16:22:54,116 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 96053ec1b36370e6b1bdd61b2f5d49cb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70341980, jitterRate=0.04817718267440796}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T16:22:54,117 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 96053ec1b36370e6b1bdd61b2f5d49cb: Writing region info on filesystem at 1733156574100Initializing all the Stores at 1733156574102 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156574102Cleaning up temporary data from old regions at 1733156574109 (+7 ms)Region opened successfully at 1733156574117 (+8 ms) 2024-12-02T16:22:54,118 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 96053ec1b36370e6b1bdd61b2f5d49cb, disabling compactions & flushes 2024-12-02T16:22:54,118 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733156574028.96053ec1b36370e6b1bdd61b2f5d49cb. 2024-12-02T16:22:54,118 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733156574028.96053ec1b36370e6b1bdd61b2f5d49cb. 2024-12-02T16:22:54,118 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733156574028.96053ec1b36370e6b1bdd61b2f5d49cb. after waiting 0 ms 2024-12-02T16:22:54,118 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733156574028.96053ec1b36370e6b1bdd61b2f5d49cb. 2024-12-02T16:22:54,118 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733156574028.96053ec1b36370e6b1bdd61b2f5d49cb. 2024-12-02T16:22:54,118 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 96053ec1b36370e6b1bdd61b2f5d49cb: Waiting for close lock at 1733156574118Disabling compacts and flushes for region at 1733156574118Disabling writes for close at 1733156574118Writing region close event to WAL at 1733156574118Closed at 1733156574118 2024-12-02T16:22:54,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741837_1013 (size=175) 2024-12-02T16:22:54,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741837_1013 (size=175) 2024-12-02T16:22:54,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741837_1013 (size=175) 2024-12-02T16:22:54,130 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T16:22:54,130 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-27690357:(num 1733156574038) 2024-12-02T16:22:54,132 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-02T16:22:54,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741839_1015 (size=469) 2024-12-02T16:22:54,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741839_1015 (size=469) 2024-12-02T16:22:54,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741839_1015 (size=469) 2024-12-02T16:22:54,173 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-02T16:22:54,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741840_1016 (size=322) 2024-12-02T16:22:54,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741840_1016 (size=322) 2024-12-02T16:22:54,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741840_1016 (size=322) 2024-12-02T16:22:54,227 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-1, size=469 (469bytes) 2024-12-02T16:22:54,228 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-02T16:22:54,228 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-02T16:22:54,228 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-1 2024-12-02T16:22:54,233 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-1 after 4ms 2024-12-02T16:22:54,267 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:22:54,268 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-1 took 42ms 2024-12-02T16:22:54,273 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-1 so closing down 2024-12-02T16:22:54,273 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T16:22:54,276 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-02T16:22:54,301 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/0000000000000000001-wal-1.temp 2024-12-02T16:22:54,302 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T16:22:54,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741841_1017 (size=469) 2024-12-02T16:22:54,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741841_1017 (size=469) 2024-12-02T16:22:54,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741841_1017 (size=469) 2024-12-02T16:22:54,314 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-02T16:22:54,317 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/0000000000000000002 2024-12-02T16:22:54,321 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 48 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-02T16:22:54,322 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-1, journal: Splitting hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-1, size=469 (469bytes) at 1733156574227Finishing writing output for hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-1 so closing down at 1733156574273 (+46 ms)Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/0000000000000000001-wal-1.temp at 1733156574301 (+28 ms)3 split writer threads finished at 1733156574302 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733156574314 (+12 ms)Rename recovered edits hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/0000000000000000002 at 1733156574317 (+3 ms)Processed 2 edits across 1 Regions in 48 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733156574321 (+4 ms) 2024-12-02T16:22:54,334 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-2, size=322 (322bytes) 2024-12-02T16:22:54,334 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-2 2024-12-02T16:22:54,335 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-2 after 1ms 2024-12-02T16:22:54,360 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:22:54,361 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-2 took 27ms 2024-12-02T16:22:54,364 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-2 so closing down 2024-12-02T16:22:54,364 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T16:22:54,367 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-02T16:22:54,393 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/0000000000000000002-wal-2.temp 2024-12-02T16:22:54,393 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T16:22:54,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741842_1018 (size=322) 2024-12-02T16:22:54,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741842_1018 (size=322) 2024-12-02T16:22:54,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741842_1018 (size=322) 2024-12-02T16:22:54,402 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-02T16:22:54,430 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:22:54,432 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-12-02T16:22:54,434 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 73 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-02T16:22:54,434 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-2, journal: Splitting hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-2, size=322 (322bytes) at 1733156574334Finishing writing output for hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-2 so closing down at 1733156574364 (+30 ms)Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/0000000000000000002-wal-2.temp at 1733156574393 (+29 ms)3 split writer threads finished at 1733156574393Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733156574402 (+9 ms)Processed 1 edits across 1 Regions in 73 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733156574434 (+32 ms) 2024-12-02T16:22:54,434 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:22:54,437 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:22:54,450 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal.1733156574438, exclude list is [], retry=0 2024-12-02T16:22:54,453 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:22:54,454 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:22:54,454 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:22:54,480 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal.1733156574438 2024-12-02T16:22:54,481 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:35029:35029)] 2024-12-02T16:22:54,481 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 96053ec1b36370e6b1bdd61b2f5d49cb, NAME => 'testReplayEditsWrittenIntoWAL,,1733156574028.96053ec1b36370e6b1bdd61b2f5d49cb.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:22:54,481 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733156574028.96053ec1b36370e6b1bdd61b2f5d49cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:22:54,482 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 96053ec1b36370e6b1bdd61b2f5d49cb 2024-12-02T16:22:54,482 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 96053ec1b36370e6b1bdd61b2f5d49cb 2024-12-02T16:22:54,484 INFO [StoreOpener-96053ec1b36370e6b1bdd61b2f5d49cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 96053ec1b36370e6b1bdd61b2f5d49cb 2024-12-02T16:22:54,485 INFO [StoreOpener-96053ec1b36370e6b1bdd61b2f5d49cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96053ec1b36370e6b1bdd61b2f5d49cb columnFamilyName a 2024-12-02T16:22:54,485 DEBUG [StoreOpener-96053ec1b36370e6b1bdd61b2f5d49cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:54,486 INFO [StoreOpener-96053ec1b36370e6b1bdd61b2f5d49cb-1 {}] regionserver.HStore(327): Store=96053ec1b36370e6b1bdd61b2f5d49cb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:22:54,487 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 96053ec1b36370e6b1bdd61b2f5d49cb 2024-12-02T16:22:54,488 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb 2024-12-02T16:22:54,490 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb 2024-12-02T16:22:54,492 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/0000000000000000002 2024-12-02T16:22:54,516 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:22:54,522 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/0000000000000000002 2024-12-02T16:22:54,526 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 96053ec1b36370e6b1bdd61b2f5d49cb 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-02T16:22:54,573 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/.tmp/a/fa32438a108f403d8079548f6d3d4079 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733156574130/Put/seqid=0 2024-12-02T16:22:54,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741844_1020 (size=5170) 2024-12-02T16:22:54,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741844_1020 (size=5170) 2024-12-02T16:22:54,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741844_1020 (size=5170) 2024-12-02T16:22:54,587 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/.tmp/a/fa32438a108f403d8079548f6d3d4079 2024-12-02T16:22:54,627 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/.tmp/a/fa32438a108f403d8079548f6d3d4079 as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/a/fa32438a108f403d8079548f6d3d4079 2024-12-02T16:22:54,636 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/a/fa32438a108f403d8079548f6d3d4079, entries=2, sequenceid=2, filesize=5.0 K 2024-12-02T16:22:54,642 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 96053ec1b36370e6b1bdd61b2f5d49cb in 116ms, sequenceid=2, compaction requested=false; wal=null 2024-12-02T16:22:54,643 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/0000000000000000002 2024-12-02T16:22:54,644 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 96053ec1b36370e6b1bdd61b2f5d49cb 2024-12-02T16:22:54,644 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 96053ec1b36370e6b1bdd61b2f5d49cb 2024-12-02T16:22:54,647 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 96053ec1b36370e6b1bdd61b2f5d49cb 2024-12-02T16:22:54,650 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/96053ec1b36370e6b1bdd61b2f5d49cb/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-02T16:22:54,652 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 96053ec1b36370e6b1bdd61b2f5d49cb; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70098577, jitterRate=0.04455019533634186}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T16:22:54,652 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 96053ec1b36370e6b1bdd61b2f5d49cb: Writing region info on filesystem at 1733156574482Initializing all the Stores at 1733156574483 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156574484 (+1 ms)Obtaining lock to block concurrent updates at 1733156574526 (+42 ms)Preparing flush snapshotting stores in 96053ec1b36370e6b1bdd61b2f5d49cb at 1733156574526Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733156574028.96053ec1b36370e6b1bdd61b2f5d49cb., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733156574529 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733156574028.96053ec1b36370e6b1bdd61b2f5d49cb. at 1733156574529Flushing 96053ec1b36370e6b1bdd61b2f5d49cb/a: creating writer at 1733156574531 (+2 ms)Flushing 96053ec1b36370e6b1bdd61b2f5d49cb/a: appending metadata at 1733156574563 (+32 ms)Flushing 96053ec1b36370e6b1bdd61b2f5d49cb/a: closing flushed file at 1733156574566 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62b352: reopening flushed file at 1733156574625 (+59 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 96053ec1b36370e6b1bdd61b2f5d49cb in 116ms, sequenceid=2, compaction requested=false; wal=null at 1733156574642 (+17 ms)Cleaning up temporary data from old regions at 1733156574644 (+2 ms)Region opened successfully at 1733156574652 (+8 ms) 2024-12-02T16:22:54,676 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=374 (was 362) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:41152 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:33443/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1520450069-172.17.0.2-1733156567009:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:34220 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:34102 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1520450069-172.17.0.2-1733156567009:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1520450069-172.17.0.2-1733156567009:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:60078 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:41288 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:60094 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=275 (was 275), ProcessCount=11 (was 11), AvailableMemoryMB=6305 (was 6321) 2024-12-02T16:22:54,687 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=374, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=275, ProcessCount=11, AvailableMemoryMB=6304 2024-12-02T16:22:54,701 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:22:54,703 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:22:54,704 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T16:22:54,708 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-40301482, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/hregion-40301482, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:22:54,721 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-40301482/hregion-40301482.1733156574709, exclude list is [], retry=0 2024-12-02T16:22:54,725 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:22:54,725 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:22:54,726 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:22:54,752 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-40301482/hregion-40301482.1733156574709 2024-12-02T16:22:54,752 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:33821:33821)] 2024-12-02T16:22:54,753 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b706c485c52e0d93dd18414d2afcebf6, NAME => 'testReplayEditsWrittenIntoWAL,,1733156574702.b706c485c52e0d93dd18414d2afcebf6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33443/hbase 2024-12-02T16:22:54,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741846_1022 (size=64) 2024-12-02T16:22:54,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741846_1022 (size=64) 2024-12-02T16:22:54,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741846_1022 (size=64) 2024-12-02T16:22:54,769 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733156574702.b706c485c52e0d93dd18414d2afcebf6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:22:54,771 INFO [StoreOpener-b706c485c52e0d93dd18414d2afcebf6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b706c485c52e0d93dd18414d2afcebf6 2024-12-02T16:22:54,773 INFO [StoreOpener-b706c485c52e0d93dd18414d2afcebf6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b706c485c52e0d93dd18414d2afcebf6 columnFamilyName a 2024-12-02T16:22:54,773 DEBUG [StoreOpener-b706c485c52e0d93dd18414d2afcebf6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:54,775 INFO [StoreOpener-b706c485c52e0d93dd18414d2afcebf6-1 {}] regionserver.HStore(327): Store=b706c485c52e0d93dd18414d2afcebf6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:22:54,775 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b706c485c52e0d93dd18414d2afcebf6 2024-12-02T16:22:54,776 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6 2024-12-02T16:22:54,777 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6 2024-12-02T16:22:54,778 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b706c485c52e0d93dd18414d2afcebf6 2024-12-02T16:22:54,778 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b706c485c52e0d93dd18414d2afcebf6 2024-12-02T16:22:54,781 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b706c485c52e0d93dd18414d2afcebf6 2024-12-02T16:22:54,785 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T16:22:54,785 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b706c485c52e0d93dd18414d2afcebf6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68395992, jitterRate=0.019179701805114746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T16:22:54,786 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b706c485c52e0d93dd18414d2afcebf6: Writing region info on filesystem at 1733156574769Initializing all the Stores at 1733156574770 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156574770Cleaning up temporary data from old regions at 1733156574778 (+8 ms)Region opened successfully at 1733156574786 (+8 ms) 2024-12-02T16:22:54,786 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b706c485c52e0d93dd18414d2afcebf6, disabling compactions & flushes 2024-12-02T16:22:54,786 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733156574702.b706c485c52e0d93dd18414d2afcebf6. 2024-12-02T16:22:54,786 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733156574702.b706c485c52e0d93dd18414d2afcebf6. 2024-12-02T16:22:54,786 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733156574702.b706c485c52e0d93dd18414d2afcebf6. after waiting 0 ms 2024-12-02T16:22:54,786 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733156574702.b706c485c52e0d93dd18414d2afcebf6. 2024-12-02T16:22:54,786 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733156574702.b706c485c52e0d93dd18414d2afcebf6. 2024-12-02T16:22:54,786 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b706c485c52e0d93dd18414d2afcebf6: Waiting for close lock at 1733156574786Disabling compacts and flushes for region at 1733156574786Disabling writes for close at 1733156574786Writing region close event to WAL at 1733156574786Closed at 1733156574786 2024-12-02T16:22:54,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741845_1021 (size=175) 2024-12-02T16:22:54,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741845_1021 (size=175) 2024-12-02T16:22:54,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741845_1021 (size=175) 2024-12-02T16:22:54,793 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T16:22:54,793 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-40301482:(num 1733156574709) 2024-12-02T16:22:54,794 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-02T16:22:54,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741847_1023 (size=469) 2024-12-02T16:22:54,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741847_1023 (size=469) 2024-12-02T16:22:54,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741847_1023 (size=469) 2024-12-02T16:22:54,829 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-02T16:22:54,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741848_1024 (size=322) 2024-12-02T16:22:54,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741848_1024 (size=322) 2024-12-02T16:22:54,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741848_1024 (size=322) 2024-12-02T16:22:54,877 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-2, size=322 (322bytes) 2024-12-02T16:22:54,877 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-2 2024-12-02T16:22:54,878 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-2 after 1ms 2024-12-02T16:22:54,903 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:22:54,903 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-2 took 27ms 2024-12-02T16:22:54,906 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-2 so closing down 2024-12-02T16:22:54,906 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T16:22:54,908 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-02T16:22:54,931 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000002-wal-2.temp 2024-12-02T16:22:54,932 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T16:22:54,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741849_1025 (size=322) 2024-12-02T16:22:54,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741849_1025 (size=322) 2024-12-02T16:22:54,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741849_1025 (size=322) 2024-12-02T16:22:54,941 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-02T16:22:54,943 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000002 2024-12-02T16:22:54,943 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-02T16:22:54,943 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-2, journal: Splitting hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-2, size=322 (322bytes) at 1733156574877Finishing writing output for hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-2 so closing down at 1733156574906 (+29 ms)Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000002-wal-2.temp at 1733156574931 (+25 ms)3 split writer threads finished at 1733156574932 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733156574941 (+9 ms)Rename recovered edits hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000002 at 1733156574943 (+2 ms)Processed 1 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733156574943 2024-12-02T16:22:54,957 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-1, size=469 (469bytes) 2024-12-02T16:22:54,958 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-1 2024-12-02T16:22:54,958 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-1 after 0ms 2024-12-02T16:22:54,983 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:22:54,983 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-1 took 26ms 2024-12-02T16:22:54,986 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-1 so closing down 2024-12-02T16:22:54,986 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T16:22:54,988 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-02T16:22:55,012 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000001-wal-1.temp 2024-12-02T16:22:55,013 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T16:22:55,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741850_1026 (size=469) 2024-12-02T16:22:55,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741850_1026 (size=469) 2024-12-02T16:22:55,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741850_1026 (size=469) 2024-12-02T16:22:55,021 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-02T16:22:55,048 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:22:55,051 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000002, length=322 2024-12-02T16:22:55,053 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000002 2024-12-02T16:22:55,053 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-02T16:22:55,053 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-1, journal: Splitting hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-1, size=469 (469bytes) at 1733156574958Finishing writing output for hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-1 so closing down at 1733156574986 (+28 ms)Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000001-wal-1.temp at 1733156575012 (+26 ms)3 split writer threads finished at 1733156575013 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733156575021 (+8 ms)Rename recovered edits hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000002 at 1733156575053 (+32 ms)Processed 2 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733156575053 2024-12-02T16:22:55,054 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:22:55,056 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:22:55,068 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal.1733156575056, exclude list is [], retry=0 2024-12-02T16:22:55,071 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:22:55,072 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:22:55,072 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:22:55,099 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal.1733156575056 2024-12-02T16:22:55,099 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:35029:35029)] 2024-12-02T16:22:55,099 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b706c485c52e0d93dd18414d2afcebf6, NAME => 'testReplayEditsWrittenIntoWAL,,1733156574702.b706c485c52e0d93dd18414d2afcebf6.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:22:55,099 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733156574702.b706c485c52e0d93dd18414d2afcebf6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:22:55,100 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b706c485c52e0d93dd18414d2afcebf6 2024-12-02T16:22:55,100 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b706c485c52e0d93dd18414d2afcebf6 2024-12-02T16:22:55,102 INFO [StoreOpener-b706c485c52e0d93dd18414d2afcebf6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b706c485c52e0d93dd18414d2afcebf6 2024-12-02T16:22:55,104 INFO [StoreOpener-b706c485c52e0d93dd18414d2afcebf6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b706c485c52e0d93dd18414d2afcebf6 columnFamilyName a 2024-12-02T16:22:55,104 DEBUG [StoreOpener-b706c485c52e0d93dd18414d2afcebf6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:55,105 INFO [StoreOpener-b706c485c52e0d93dd18414d2afcebf6-1 {}] regionserver.HStore(327): Store=b706c485c52e0d93dd18414d2afcebf6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:22:55,105 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b706c485c52e0d93dd18414d2afcebf6 2024-12-02T16:22:55,106 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6 2024-12-02T16:22:55,108 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6 2024-12-02T16:22:55,109 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000002 2024-12-02T16:22:55,134 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:22:55,136 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000002 2024-12-02T16:22:55,136 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b706c485c52e0d93dd18414d2afcebf6 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-02T16:22:55,151 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/.tmp/a/621a71619ec347a99a1e448b04798638 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733156574793/Put/seqid=0 2024-12-02T16:22:55,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741852_1028 (size=5170) 2024-12-02T16:22:55,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741852_1028 (size=5170) 2024-12-02T16:22:55,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741852_1028 (size=5170) 2024-12-02T16:22:55,160 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/.tmp/a/621a71619ec347a99a1e448b04798638 2024-12-02T16:22:55,169 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/.tmp/a/621a71619ec347a99a1e448b04798638 as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/a/621a71619ec347a99a1e448b04798638 2024-12-02T16:22:55,179 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/a/621a71619ec347a99a1e448b04798638, entries=2, sequenceid=2, filesize=5.0 K 2024-12-02T16:22:55,179 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for b706c485c52e0d93dd18414d2afcebf6 in 43ms, sequenceid=2, compaction requested=false; wal=null 2024-12-02T16:22:55,180 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/0000000000000000002 2024-12-02T16:22:55,181 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b706c485c52e0d93dd18414d2afcebf6 2024-12-02T16:22:55,181 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b706c485c52e0d93dd18414d2afcebf6 2024-12-02T16:22:55,183 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b706c485c52e0d93dd18414d2afcebf6 2024-12-02T16:22:55,186 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/b706c485c52e0d93dd18414d2afcebf6/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-02T16:22:55,188 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b706c485c52e0d93dd18414d2afcebf6; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62752566, jitterRate=-0.06491389870643616}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T16:22:55,188 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b706c485c52e0d93dd18414d2afcebf6: Writing region info on filesystem at 1733156575100Initializing all the Stores at 1733156575101 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156575101Obtaining lock to block concurrent updates at 1733156575136 (+35 ms)Preparing flush snapshotting stores in b706c485c52e0d93dd18414d2afcebf6 at 1733156575136Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733156574702.b706c485c52e0d93dd18414d2afcebf6., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733156575136Flushing stores of testReplayEditsWrittenIntoWAL,,1733156574702.b706c485c52e0d93dd18414d2afcebf6. at 1733156575136Flushing b706c485c52e0d93dd18414d2afcebf6/a: creating writer at 1733156575136Flushing b706c485c52e0d93dd18414d2afcebf6/a: appending metadata at 1733156575150 (+14 ms)Flushing b706c485c52e0d93dd18414d2afcebf6/a: closing flushed file at 1733156575150Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a615a68: reopening flushed file at 1733156575167 (+17 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for b706c485c52e0d93dd18414d2afcebf6 in 43ms, sequenceid=2, compaction requested=false; wal=null at 1733156575179 (+12 ms)Cleaning up temporary data from old regions at 1733156575181 (+2 ms)Region opened successfully at 1733156575188 (+7 ms) 2024-12-02T16:22:55,204 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=384 (was 374) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:41152 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:54392 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:43488 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:34102 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:58548 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1520450069-172.17.0.2-1733156567009:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1520450069-172.17.0.2-1733156567009:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1520450069-172.17.0.2-1733156567009:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=275 (was 275), ProcessCount=11 (was 11), AvailableMemoryMB=6293 (was 6304) 2024-12-02T16:22:55,216 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=384, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=275, ProcessCount=11, AvailableMemoryMB=6292 2024-12-02T16:22:55,235 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:22:55,238 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:22:55,239 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T16:22:55,242 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-51264900, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/hregion-51264900, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:22:55,255 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-51264900/hregion-51264900.1733156575243, exclude list is [], retry=0 2024-12-02T16:22:55,258 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:22:55,259 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:22:55,259 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:22:55,282 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-51264900/hregion-51264900.1733156575243 2024-12-02T16:22:55,282 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:43815:43815)] 2024-12-02T16:22:55,282 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 43712fc29289f2b28503ae9de0b1bfbe, NAME => 'testReplayEditsWrittenIntoWAL,,1733156575236.43712fc29289f2b28503ae9de0b1bfbe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33443/hbase 2024-12-02T16:22:55,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741854_1030 (size=64) 2024-12-02T16:22:55,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741854_1030 (size=64) 2024-12-02T16:22:55,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741854_1030 (size=64) 2024-12-02T16:22:55,293 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733156575236.43712fc29289f2b28503ae9de0b1bfbe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:22:55,295 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:22:55,297 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43712fc29289f2b28503ae9de0b1bfbe columnFamilyName a 2024-12-02T16:22:55,297 DEBUG [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:55,298 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] regionserver.HStore(327): Store=43712fc29289f2b28503ae9de0b1bfbe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:22:55,298 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:22:55,300 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43712fc29289f2b28503ae9de0b1bfbe columnFamilyName b 2024-12-02T16:22:55,300 DEBUG [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:55,300 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] regionserver.HStore(327): Store=43712fc29289f2b28503ae9de0b1bfbe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:22:55,301 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:22:55,303 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43712fc29289f2b28503ae9de0b1bfbe columnFamilyName c 2024-12-02T16:22:55,303 DEBUG [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:55,304 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] regionserver.HStore(327): Store=43712fc29289f2b28503ae9de0b1bfbe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:22:55,304 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:22:55,305 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:22:55,305 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:22:55,306 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:22:55,306 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:22:55,307 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:22:55,308 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:22:55,310 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T16:22:55,311 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 43712fc29289f2b28503ae9de0b1bfbe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63833066, jitterRate=-0.048813194036483765}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:22:55,311 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 43712fc29289f2b28503ae9de0b1bfbe: Writing region info on filesystem at 1733156575293Initializing all the Stores at 1733156575295 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156575295Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156575295Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156575295Cleaning up temporary data from old regions at 1733156575306 (+11 ms)Region opened successfully at 1733156575311 (+5 ms) 2024-12-02T16:22:55,311 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 43712fc29289f2b28503ae9de0b1bfbe, disabling compactions & flushes 2024-12-02T16:22:55,311 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733156575236.43712fc29289f2b28503ae9de0b1bfbe. 2024-12-02T16:22:55,311 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733156575236.43712fc29289f2b28503ae9de0b1bfbe. 2024-12-02T16:22:55,312 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733156575236.43712fc29289f2b28503ae9de0b1bfbe. after waiting 0 ms 2024-12-02T16:22:55,312 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733156575236.43712fc29289f2b28503ae9de0b1bfbe. 2024-12-02T16:22:55,312 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733156575236.43712fc29289f2b28503ae9de0b1bfbe. 2024-12-02T16:22:55,312 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 43712fc29289f2b28503ae9de0b1bfbe: Waiting for close lock at 1733156575311Disabling compacts and flushes for region at 1733156575311Disabling writes for close at 1733156575312 (+1 ms)Writing region close event to WAL at 1733156575312Closed at 1733156575312 2024-12-02T16:22:55,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741853_1029 (size=175) 2024-12-02T16:22:55,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741853_1029 (size=175) 2024-12-02T16:22:55,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741853_1029 (size=175) 2024-12-02T16:22:55,318 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T16:22:55,318 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-51264900:(num 1733156575243) 2024-12-02T16:22:55,318 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:22:55,321 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:22:55,335 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321, exclude list is [], retry=0 2024-12-02T16:22:55,338 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:22:55,339 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:22:55,339 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:22:55,364 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321 2024-12-02T16:22:55,365 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:43815:43815)] 2024-12-02T16:22:55,566 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321, size=0 (0bytes) 2024-12-02T16:22:55,566 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321 might be still open, length is 0 2024-12-02T16:22:55,566 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321 2024-12-02T16:22:55,567 WARN [IPC Server handler 2 on default port 33443 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-02T16:22:55,567 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321 after 1ms 2024-12-02T16:22:55,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741831_1007 (size=1321) 2024-12-02T16:22:55,829 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:43518 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33271:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43518 dst: /127.0.0.1:33271 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33271 remote=/127.0.0.1:43518]. Total timeout mills is 60000, 59708 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:22:55,830 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:54436 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38643:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54436 dst: /127.0.0.1:38643 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:22:55,830 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:58582 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33041:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58582 dst: /127.0.0.1:33041 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:22:55,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741855_1032 (size=497775) 2024-12-02T16:22:55,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741855_1032 (size=497775) 2024-12-02T16:22:55,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741855_1032 (size=497775) 2024-12-02T16:22:59,090 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T16:22:59,148 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T16:22:59,568 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321 after 4002ms 2024-12-02T16:22:59,601 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:22:59,603 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321 took 4038ms 2024-12-02T16:22:59,607 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733156575321.temp 2024-12-02T16:22:59,633 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/recovered.edits/0000000000000000001-wal.1733156575321.temp 2024-12-02T16:22:59,703 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321; continuing. 2024-12-02T16:22:59,703 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321 so closing down 2024-12-02T16:22:59,703 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T16:22:59,703 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T16:22:59,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741856_1033 (size=497783) 2024-12-02T16:22:59,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741856_1033 (size=497783) 2024-12-02T16:22:59,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741856_1033 (size=497783) 2024-12-02T16:22:59,708 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/recovered.edits/0000000000000000001-wal.1733156575321.temp (wrote 3002 edits, skipped 0 edits in 42 ms) 2024-12-02T16:22:59,710 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/recovered.edits/0000000000000000001-wal.1733156575321.temp to hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/recovered.edits/0000000000000003002 2024-12-02T16:22:59,710 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 107 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321, size=0, length=0, corrupted=false, cancelled=false 2024-12-02T16:22:59,710 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321, journal: Splitting hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321, size=0 (0bytes) at 1733156575566Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/recovered.edits/0000000000000000001-wal.1733156575321.temp at 1733156579633 (+4067 ms)Split 1024 edits, skipped 0 edits. at 1733156579663 (+30 ms)Split 2048 edits, skipped 0 edits. at 1733156579684 (+21 ms)Finishing writing output for hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321 so closing down at 1733156579703 (+19 ms)3 split writer threads finished at 1733156579703Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/recovered.edits/0000000000000000001-wal.1733156575321.temp (wrote 3002 edits, skipped 0 edits in 42 ms) at 1733156579708 (+5 ms)Rename recovered edits hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/recovered.edits/0000000000000000001-wal.1733156575321.temp to hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/recovered.edits/0000000000000003002 at 1733156579710 (+2 ms)Processed 3002 edits across 1 Regions in 107 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321, size=0, length=0, corrupted=false, cancelled=false at 1733156579710 2024-12-02T16:22:59,713 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321 to hdfs://localhost:33443/hbase/oldWALs/wal.1733156575321 2024-12-02T16:22:59,714 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/recovered.edits/0000000000000003002 2024-12-02T16:22:59,715 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:22:59,717 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:22:59,736 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156579718, exclude list is [], retry=0 2024-12-02T16:22:59,739 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:22:59,740 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:22:59,740 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:22:59,765 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156579718 2024-12-02T16:22:59,765 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:33821:33821)] 2024-12-02T16:22:59,766 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733156575236.43712fc29289f2b28503ae9de0b1bfbe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:22:59,768 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:22:59,769 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43712fc29289f2b28503ae9de0b1bfbe columnFamilyName a 2024-12-02T16:22:59,769 DEBUG [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:59,770 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] regionserver.HStore(327): Store=43712fc29289f2b28503ae9de0b1bfbe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:22:59,770 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:22:59,771 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43712fc29289f2b28503ae9de0b1bfbe columnFamilyName b 2024-12-02T16:22:59,771 DEBUG [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:59,772 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] regionserver.HStore(327): Store=43712fc29289f2b28503ae9de0b1bfbe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:22:59,772 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:22:59,773 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43712fc29289f2b28503ae9de0b1bfbe columnFamilyName c 2024-12-02T16:22:59,773 DEBUG [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:22:59,773 INFO [StoreOpener-43712fc29289f2b28503ae9de0b1bfbe-1 {}] regionserver.HStore(327): Store=43712fc29289f2b28503ae9de0b1bfbe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:22:59,773 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:22:59,774 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:22:59,777 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:22:59,778 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/recovered.edits/0000000000000003002 2024-12-02T16:22:59,810 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:22:59,852 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-02T16:23:00,186 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 43712fc29289f2b28503ae9de0b1bfbe 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-02T16:23:00,227 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/a/6e59a0abcdbb4309b9ed0f24f13103c5 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733156575372/Put/seqid=0 2024-12-02T16:23:00,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741858_1035 (size=50463) 2024-12-02T16:23:00,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741858_1035 (size=50463) 2024-12-02T16:23:00,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741858_1035 (size=50463) 2024-12-02T16:23:00,238 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/a/6e59a0abcdbb4309b9ed0f24f13103c5 2024-12-02T16:23:00,246 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/a/6e59a0abcdbb4309b9ed0f24f13103c5 as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/a/6e59a0abcdbb4309b9ed0f24f13103c5 2024-12-02T16:23:00,254 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/a/6e59a0abcdbb4309b9ed0f24f13103c5, entries=754, sequenceid=754, filesize=49.3 K 2024-12-02T16:23:00,255 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 43712fc29289f2b28503ae9de0b1bfbe in 68ms, sequenceid=754, compaction requested=false; wal=null 2024-12-02T16:23:00,275 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-02T16:23:00,275 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 43712fc29289f2b28503ae9de0b1bfbe 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-02T16:23:00,284 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/a/09cbeb5043e14b4897cd04f16053ee14 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733156575420/Put/seqid=0 2024-12-02T16:23:00,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741859_1036 (size=20072) 2024-12-02T16:23:00,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741859_1036 (size=20072) 2024-12-02T16:23:00,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741859_1036 (size=20072) 2024-12-02T16:23:00,298 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/a/09cbeb5043e14b4897cd04f16053ee14 2024-12-02T16:23:00,321 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/b/339354e5eebb4f53911a66adff6277bd is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733156575442/Put/seqid=0 2024-12-02T16:23:00,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741860_1037 (size=35835) 2024-12-02T16:23:00,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741860_1037 (size=35835) 2024-12-02T16:23:00,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741860_1037 (size=35835) 2024-12-02T16:23:00,330 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/b/339354e5eebb4f53911a66adff6277bd 2024-12-02T16:23:00,338 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/a/09cbeb5043e14b4897cd04f16053ee14 as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/a/09cbeb5043e14b4897cd04f16053ee14 2024-12-02T16:23:00,347 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/a/09cbeb5043e14b4897cd04f16053ee14, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-02T16:23:00,349 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/b/339354e5eebb4f53911a66adff6277bd as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/b/339354e5eebb4f53911a66adff6277bd 2024-12-02T16:23:00,356 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/b/339354e5eebb4f53911a66adff6277bd, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-02T16:23:00,357 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 43712fc29289f2b28503ae9de0b1bfbe in 82ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-02T16:23:00,369 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-02T16:23:00,370 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 43712fc29289f2b28503ae9de0b1bfbe 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-02T16:23:00,377 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/b/c742892507cd4d4eb20ae80fed674f02 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733156575461/Put/seqid=0 2024-12-02T16:23:00,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741861_1038 (size=35082) 2024-12-02T16:23:00,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741861_1038 (size=35082) 2024-12-02T16:23:00,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741861_1038 (size=35082) 2024-12-02T16:23:00,387 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/b/c742892507cd4d4eb20ae80fed674f02 2024-12-02T16:23:00,411 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/c/ef5c44eafc23482594ff0caff7aa460b is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733156575491/Put/seqid=0 2024-12-02T16:23:00,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741862_1039 (size=20825) 2024-12-02T16:23:00,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741862_1039 (size=20825) 2024-12-02T16:23:00,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741862_1039 (size=20825) 2024-12-02T16:23:00,420 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/c/ef5c44eafc23482594ff0caff7aa460b 2024-12-02T16:23:00,428 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/b/c742892507cd4d4eb20ae80fed674f02 as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/b/c742892507cd4d4eb20ae80fed674f02 2024-12-02T16:23:00,436 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/b/c742892507cd4d4eb20ae80fed674f02, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-02T16:23:00,438 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/c/ef5c44eafc23482594ff0caff7aa460b as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/c/ef5c44eafc23482594ff0caff7aa460b 2024-12-02T16:23:00,445 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/c/ef5c44eafc23482594ff0caff7aa460b, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-02T16:23:00,446 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 43712fc29289f2b28503ae9de0b1bfbe in 76ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-02T16:23:00,459 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733156575534/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733156575236.43712fc29289f2b28503ae9de0b1bfbe. 2024-12-02T16:23:00,463 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/recovered.edits/0000000000000003002 2024-12-02T16:23:00,463 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-02T16:23:00,464 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 43712fc29289f2b28503ae9de0b1bfbe 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-02T16:23:00,473 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/c/18948594f5114fd38d5710bf87ace05d is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733156575496/Put/seqid=0 2024-12-02T16:23:00,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741863_1040 (size=50301) 2024-12-02T16:23:00,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741863_1040 (size=50301) 2024-12-02T16:23:00,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741863_1040 (size=50301) 2024-12-02T16:23:00,484 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/c/18948594f5114fd38d5710bf87ace05d 2024-12-02T16:23:00,490 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 18948594f5114fd38d5710bf87ace05d 2024-12-02T16:23:00,492 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/.tmp/c/18948594f5114fd38d5710bf87ace05d as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/c/18948594f5114fd38d5710bf87ace05d 2024-12-02T16:23:00,499 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 18948594f5114fd38d5710bf87ace05d 2024-12-02T16:23:00,499 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/c/18948594f5114fd38d5710bf87ace05d, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-02T16:23:00,500 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 43712fc29289f2b28503ae9de0b1bfbe in 37ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-02T16:23:00,501 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/recovered.edits/0000000000000003002 2024-12-02T16:23:00,502 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:23:00,502 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:23:00,503 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T16:23:00,505 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 43712fc29289f2b28503ae9de0b1bfbe 2024-12-02T16:23:00,508 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenIntoWAL/43712fc29289f2b28503ae9de0b1bfbe/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-02T16:23:00,510 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 43712fc29289f2b28503ae9de0b1bfbe; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61113536, jitterRate=-0.08933734893798828}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T16:23:00,510 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 43712fc29289f2b28503ae9de0b1bfbe: Writing region info on filesystem at 1733156579766Initializing all the Stores at 1733156579767 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156579767Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156579768 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156579768Cleaning up temporary data from old regions at 1733156580502 (+734 ms)Region opened successfully at 1733156580510 (+8 ms) 2024-12-02T16:23:00,584 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 43712fc29289f2b28503ae9de0b1bfbe, disabling compactions & flushes 2024-12-02T16:23:00,584 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733156575236.43712fc29289f2b28503ae9de0b1bfbe. 2024-12-02T16:23:00,584 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733156575236.43712fc29289f2b28503ae9de0b1bfbe. 2024-12-02T16:23:00,584 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733156575236.43712fc29289f2b28503ae9de0b1bfbe. after waiting 0 ms 2024-12-02T16:23:00,584 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733156575236.43712fc29289f2b28503ae9de0b1bfbe. 2024-12-02T16:23:00,586 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733156575236.43712fc29289f2b28503ae9de0b1bfbe. 2024-12-02T16:23:00,586 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 43712fc29289f2b28503ae9de0b1bfbe: Waiting for close lock at 1733156580583Disabling compacts and flushes for region at 1733156580584 (+1 ms)Disabling writes for close at 1733156580584Writing region close event to WAL at 1733156580586 (+2 ms)Closed at 1733156580586 2024-12-02T16:23:00,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741857_1034 (size=175) 2024-12-02T16:23:00,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741857_1034 (size=175) 2024-12-02T16:23:00,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741857_1034 (size=175) 2024-12-02T16:23:00,594 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T16:23:00,594 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733156579718) 2024-12-02T16:23:00,610 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=399 (was 384) Potentially hanging thread: IPC Client (1685805399) connection to localhost/127.0.0.1:38767 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1685805399) connection to localhost/127.0.0.1:38325 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38325 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1815045869_22 at /127.0.0.1:54538 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38767 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:33443 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1815045869_22 at /127.0.0.1:58598 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1815045869_22 at /127.0.0.1:54488 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1685805399) connection to localhost/127.0.0.1:33443 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1815045869_22 at /127.0.0.1:43554 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@4dc7c0c2[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=253 (was 275), ProcessCount=11 (was 11), AvailableMemoryMB=6184 (was 6292) 2024-12-02T16:23:00,624 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=399, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=253, ProcessCount=11, AvailableMemoryMB=6181 2024-12-02T16:23:00,645 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:23:00,648 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:23:00,649 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T16:23:00,652 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-92608755, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/hregion-92608755, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:00,668 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-92608755/hregion-92608755.1733156580653, exclude list is [], retry=0 2024-12-02T16:23:00,672 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:00,672 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:00,672 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:00,696 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-92608755/hregion-92608755.1733156580653 2024-12-02T16:23:00,697 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:43815:43815)] 2024-12-02T16:23:00,697 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 430eac8954ff8d12eccbd6179d2eae2f, NAME => 'test2727,,1733156580646.430eac8954ff8d12eccbd6179d2eae2f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33443/hbase 2024-12-02T16:23:00,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741865_1042 (size=43) 2024-12-02T16:23:00,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741865_1042 (size=43) 2024-12-02T16:23:00,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741865_1042 (size=43) 2024-12-02T16:23:00,707 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733156580646.430eac8954ff8d12eccbd6179d2eae2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:00,709 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:00,711 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 430eac8954ff8d12eccbd6179d2eae2f columnFamilyName a 2024-12-02T16:23:00,711 DEBUG [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:00,711 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] regionserver.HStore(327): Store=430eac8954ff8d12eccbd6179d2eae2f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:00,712 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:00,714 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 430eac8954ff8d12eccbd6179d2eae2f columnFamilyName b 2024-12-02T16:23:00,714 DEBUG [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:00,714 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] regionserver.HStore(327): Store=430eac8954ff8d12eccbd6179d2eae2f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:00,715 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:00,716 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 430eac8954ff8d12eccbd6179d2eae2f columnFamilyName c 2024-12-02T16:23:00,716 DEBUG [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:00,717 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] regionserver.HStore(327): Store=430eac8954ff8d12eccbd6179d2eae2f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:00,717 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:00,718 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:00,719 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:00,720 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:00,720 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:00,720 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:00,722 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:00,725 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T16:23:00,725 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 430eac8954ff8d12eccbd6179d2eae2f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65498612, jitterRate=-0.023994624614715576}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:00,726 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 430eac8954ff8d12eccbd6179d2eae2f: Writing region info on filesystem at 1733156580707Initializing all the Stores at 1733156580708 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156580708Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156580708Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156580709 (+1 ms)Cleaning up temporary data from old regions at 1733156580720 (+11 ms)Region opened successfully at 1733156580726 (+6 ms) 2024-12-02T16:23:00,727 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 430eac8954ff8d12eccbd6179d2eae2f, disabling compactions & flushes 2024-12-02T16:23:00,727 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733156580646.430eac8954ff8d12eccbd6179d2eae2f. 2024-12-02T16:23:00,727 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733156580646.430eac8954ff8d12eccbd6179d2eae2f. 2024-12-02T16:23:00,727 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733156580646.430eac8954ff8d12eccbd6179d2eae2f. after waiting 0 ms 2024-12-02T16:23:00,727 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733156580646.430eac8954ff8d12eccbd6179d2eae2f. 2024-12-02T16:23:00,727 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733156580646.430eac8954ff8d12eccbd6179d2eae2f. 2024-12-02T16:23:00,727 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 430eac8954ff8d12eccbd6179d2eae2f: Waiting for close lock at 1733156580727Disabling compacts and flushes for region at 1733156580727Disabling writes for close at 1733156580727Writing region close event to WAL at 1733156580727Closed at 1733156580727 2024-12-02T16:23:00,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741864_1041 (size=175) 2024-12-02T16:23:00,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741864_1041 (size=175) 2024-12-02T16:23:00,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741864_1041 (size=175) 2024-12-02T16:23:00,734 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T16:23:00,734 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-92608755:(num 1733156580653) 2024-12-02T16:23:00,734 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:23:00,737 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:00,757 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156580738, exclude list is [], retry=0 2024-12-02T16:23:00,761 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:00,762 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:00,762 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:00,825 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156580738 2024-12-02T16:23:00,826 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:43815:43815)] 2024-12-02T16:23:00,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741866_1043 (size=371388) 2024-12-02T16:23:00,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741866_1043 (size=371388) 2024-12-02T16:23:00,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741866_1043 (size=371388) 2024-12-02T16:23:00,959 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156580738 not finished, retry = 0 2024-12-02T16:23:01,010 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T16:23:01,011 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-02T16:23:01,012 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T16:23:01,012 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-02T16:23:01,013 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-02T16:23:01,013 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-02T16:23:01,013 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-02T16:23:01,014 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-02T16:23:01,074 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156580738, size=362.7 K (371388bytes) 2024-12-02T16:23:01,074 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156580738 2024-12-02T16:23:01,075 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156580738 after 1ms 2024-12-02T16:23:01,100 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:01,102 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156580738 took 28ms 2024-12-02T16:23:01,105 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733156580738.temp 2024-12-02T16:23:01,136 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156580738 so closing down 2024-12-02T16:23:01,136 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T16:23:01,147 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000000001-wal.1733156580738.temp 2024-12-02T16:23:01,163 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T16:23:01,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741867_1044 (size=371388) 2024-12-02T16:23:01,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741867_1044 (size=371388) 2024-12-02T16:23:01,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741867_1044 (size=371388) 2024-12-02T16:23:01,168 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000000001-wal.1733156580738.temp (wrote 3000 edits, skipped 0 edits in 15 ms) 2024-12-02T16:23:01,170 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000000001-wal.1733156580738.temp to hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000003000 2024-12-02T16:23:01,170 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156580738, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-12-02T16:23:01,170 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156580738, journal: Splitting hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156580738, size=362.7 K (371388bytes) at 1733156581074Split 1024 edits, skipped 0 edits. at 1733156581114 (+40 ms)Split 2048 edits, skipped 0 edits. at 1733156581126 (+12 ms)Finishing writing output for hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156580738 so closing down at 1733156581136 (+10 ms)Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000000001-wal.1733156580738.temp at 1733156581147 (+11 ms)3 split writer threads finished at 1733156581163 (+16 ms)Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000000001-wal.1733156580738.temp (wrote 3000 edits, skipped 0 edits in 15 ms) at 1733156581168 (+5 ms)Rename recovered edits hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000000001-wal.1733156580738.temp to hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000003000 at 1733156581170 (+2 ms)Processed 3000 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156580738, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1733156581170 2024-12-02T16:23:01,173 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156580738 to hdfs://localhost:33443/hbase/oldWALs/wal.1733156580738 2024-12-02T16:23:01,175 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000003000 2024-12-02T16:23:01,175 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:23:01,178 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:01,198 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156581179, exclude list is [], retry=0 2024-12-02T16:23:01,202 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:01,202 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:01,203 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:01,240 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156581179 2024-12-02T16:23:01,241 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:33821:33821)] 2024-12-02T16:23:01,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741868_1045 (size=371515) 2024-12-02T16:23:01,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741868_1045 (size=371515) 2024-12-02T16:23:01,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741868_1045 (size=371515) 2024-12-02T16:23:01,384 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156581179, size=362.8 K (371515bytes) 2024-12-02T16:23:01,384 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156581179 2024-12-02T16:23:01,385 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156581179 after 1ms 2024-12-02T16:23:01,416 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:01,418 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156581179 took 34ms 2024-12-02T16:23:01,422 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733156581179.temp 2024-12-02T16:23:01,441 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156581179 so closing down 2024-12-02T16:23:01,441 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T16:23:01,448 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000003001-wal.1733156581179.temp 2024-12-02T16:23:01,457 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T16:23:01,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741869_1046 (size=371515) 2024-12-02T16:23:01,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741869_1046 (size=371515) 2024-12-02T16:23:01,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741869_1046 (size=371515) 2024-12-02T16:23:01,462 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000003001-wal.1733156581179.temp (wrote 3000 edits, skipped 0 edits in 9 ms) 2024-12-02T16:23:01,464 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000003001-wal.1733156581179.temp to hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000006000 2024-12-02T16:23:01,465 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156581179, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-12-02T16:23:01,465 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156581179, journal: Splitting hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156581179, size=362.8 K (371515bytes) at 1733156581384Split 1024 edits, skipped 0 edits. at 1733156581428 (+44 ms)Split 2048 edits, skipped 0 edits. at 1733156581435 (+7 ms)Finishing writing output for hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156581179 so closing down at 1733156581441 (+6 ms)Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000003001-wal.1733156581179.temp at 1733156581448 (+7 ms)3 split writer threads finished at 1733156581458 (+10 ms)Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000003001-wal.1733156581179.temp (wrote 3000 edits, skipped 0 edits in 9 ms) at 1733156581462 (+4 ms)Rename recovered edits hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000003001-wal.1733156581179.temp to hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000006000 at 1733156581464 (+2 ms)Processed 3000 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156581179, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1733156581465 (+1 ms) 2024-12-02T16:23:01,467 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156581179 to hdfs://localhost:33443/hbase/oldWALs/wal.1733156581179 2024-12-02T16:23:01,468 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000006000 2024-12-02T16:23:01,469 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:23:01,471 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/test2727-manual,16010,1733156580644, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:01,484 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156581472, exclude list is [], retry=0 2024-12-02T16:23:01,487 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:01,488 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:01,488 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:01,513 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156581472 2024-12-02T16:23:01,513 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:43815:43815)] 2024-12-02T16:23:01,514 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 430eac8954ff8d12eccbd6179d2eae2f, NAME => 'test2727,,1733156580646.430eac8954ff8d12eccbd6179d2eae2f.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:01,514 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733156580646.430eac8954ff8d12eccbd6179d2eae2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:01,514 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:01,514 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:01,516 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:01,517 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 430eac8954ff8d12eccbd6179d2eae2f columnFamilyName a 2024-12-02T16:23:01,517 DEBUG [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:01,518 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] regionserver.HStore(327): Store=430eac8954ff8d12eccbd6179d2eae2f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:01,518 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:01,520 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 430eac8954ff8d12eccbd6179d2eae2f columnFamilyName b 2024-12-02T16:23:01,520 DEBUG [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:01,521 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] regionserver.HStore(327): Store=430eac8954ff8d12eccbd6179d2eae2f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:01,521 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:01,523 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 430eac8954ff8d12eccbd6179d2eae2f columnFamilyName c 2024-12-02T16:23:01,523 DEBUG [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:01,524 INFO [StoreOpener-430eac8954ff8d12eccbd6179d2eae2f-1 {}] regionserver.HStore(327): Store=430eac8954ff8d12eccbd6179d2eae2f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:01,524 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:01,525 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:01,528 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:01,529 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000003000 2024-12-02T16:23:01,553 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:01,597 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000003000 2024-12-02T16:23:01,598 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000006000 2024-12-02T16:23:01,631 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:01,655 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000006000 2024-12-02T16:23:01,655 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 430eac8954ff8d12eccbd6179d2eae2f 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-02T16:23:01,675 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/.tmp/a/a41da80f7c8a41689a527c4fe1dc3059 is 41, key is test2727/a:100/1733156581244/Put/seqid=0 2024-12-02T16:23:01,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741871_1048 (size=84227) 2024-12-02T16:23:01,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741871_1048 (size=84227) 2024-12-02T16:23:01,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741871_1048 (size=84227) 2024-12-02T16:23:01,685 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/.tmp/a/a41da80f7c8a41689a527c4fe1dc3059 2024-12-02T16:23:01,712 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/.tmp/b/a1de06d1e0224c71a1db682efb8b3e8e is 41, key is test2727/b:100/1733156581282/Put/seqid=0 2024-12-02T16:23:01,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741872_1049 (size=84609) 2024-12-02T16:23:01,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741872_1049 (size=84609) 2024-12-02T16:23:01,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741872_1049 (size=84609) 2024-12-02T16:23:01,718 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/.tmp/b/a1de06d1e0224c71a1db682efb8b3e8e 2024-12-02T16:23:01,743 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/.tmp/c/ce1fe1edf83844ad845c09e2ee408f9a is 41, key is test2727/c:100/1733156581324/Put/seqid=0 2024-12-02T16:23:01,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741873_1050 (size=84609) 2024-12-02T16:23:01,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741873_1050 (size=84609) 2024-12-02T16:23:01,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741873_1050 (size=84609) 2024-12-02T16:23:01,750 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/.tmp/c/ce1fe1edf83844ad845c09e2ee408f9a 2024-12-02T16:23:01,759 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/.tmp/a/a41da80f7c8a41689a527c4fe1dc3059 as hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/a/a41da80f7c8a41689a527c4fe1dc3059 2024-12-02T16:23:01,767 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/a/a41da80f7c8a41689a527c4fe1dc3059, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-02T16:23:01,769 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/.tmp/b/a1de06d1e0224c71a1db682efb8b3e8e as hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/b/a1de06d1e0224c71a1db682efb8b3e8e 2024-12-02T16:23:01,779 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/b/a1de06d1e0224c71a1db682efb8b3e8e, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-02T16:23:01,784 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/.tmp/c/ce1fe1edf83844ad845c09e2ee408f9a as hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/c/ce1fe1edf83844ad845c09e2ee408f9a 2024-12-02T16:23:01,792 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/c/ce1fe1edf83844ad845c09e2ee408f9a, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-02T16:23:01,792 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 430eac8954ff8d12eccbd6179d2eae2f in 137ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-02T16:23:01,793 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000003000 2024-12-02T16:23:01,794 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/0000000000000006000 2024-12-02T16:23:01,795 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:01,795 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:01,796 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:01,799 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 430eac8954ff8d12eccbd6179d2eae2f 2024-12-02T16:23:01,802 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/test2727/430eac8954ff8d12eccbd6179d2eae2f/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-02T16:23:01,804 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 430eac8954ff8d12eccbd6179d2eae2f; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62736753, jitterRate=-0.06514953076839447}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:01,805 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 430eac8954ff8d12eccbd6179d2eae2f: Writing region info on filesystem at 1733156581514Initializing all the Stores at 1733156581515 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156581515Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156581515Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156581515Obtaining lock to block concurrent updates at 1733156581655 (+140 ms)Preparing flush snapshotting stores in 430eac8954ff8d12eccbd6179d2eae2f at 1733156581655Finished memstore snapshotting test2727,,1733156580646.430eac8954ff8d12eccbd6179d2eae2f., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733156581656 (+1 ms)Flushing stores of test2727,,1733156580646.430eac8954ff8d12eccbd6179d2eae2f. at 1733156581656Flushing 430eac8954ff8d12eccbd6179d2eae2f/a: creating writer at 1733156581656Flushing 430eac8954ff8d12eccbd6179d2eae2f/a: appending metadata at 1733156581675 (+19 ms)Flushing 430eac8954ff8d12eccbd6179d2eae2f/a: closing flushed file at 1733156581675Flushing 430eac8954ff8d12eccbd6179d2eae2f/b: creating writer at 1733156581693 (+18 ms)Flushing 430eac8954ff8d12eccbd6179d2eae2f/b: appending metadata at 1733156581710 (+17 ms)Flushing 430eac8954ff8d12eccbd6179d2eae2f/b: closing flushed file at 1733156581710Flushing 430eac8954ff8d12eccbd6179d2eae2f/c: creating writer at 1733156581725 (+15 ms)Flushing 430eac8954ff8d12eccbd6179d2eae2f/c: appending metadata at 1733156581742 (+17 ms)Flushing 430eac8954ff8d12eccbd6179d2eae2f/c: closing flushed file at 1733156581742Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a3820c7: reopening flushed file at 1733156581757 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35a9adea: reopening flushed file at 1733156581767 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f97091a: reopening flushed file at 1733156581780 (+13 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 430eac8954ff8d12eccbd6179d2eae2f in 137ms, sequenceid=6000, compaction requested=false; wal=null at 1733156581792 (+12 ms)Cleaning up temporary data from old regions at 1733156581795 (+3 ms)Region opened successfully at 1733156581804 (+9 ms) 2024-12-02T16:23:01,806 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-02T16:23:01,806 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 430eac8954ff8d12eccbd6179d2eae2f, disabling compactions & flushes 2024-12-02T16:23:01,806 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733156580646.430eac8954ff8d12eccbd6179d2eae2f. 2024-12-02T16:23:01,806 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733156580646.430eac8954ff8d12eccbd6179d2eae2f. 2024-12-02T16:23:01,806 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733156580646.430eac8954ff8d12eccbd6179d2eae2f. after waiting 0 ms 2024-12-02T16:23:01,806 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733156580646.430eac8954ff8d12eccbd6179d2eae2f. 2024-12-02T16:23:01,807 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733156580646.430eac8954ff8d12eccbd6179d2eae2f. 2024-12-02T16:23:01,807 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 430eac8954ff8d12eccbd6179d2eae2f: Waiting for close lock at 1733156581806Disabling compacts and flushes for region at 1733156581806Disabling writes for close at 1733156581806Writing region close event to WAL at 1733156581807 (+1 ms)Closed at 1733156581807 2024-12-02T16:23:01,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741870_1047 (size=175) 2024-12-02T16:23:01,812 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733156580644/wal.1733156581472 not finished, retry = 0 2024-12-02T16:23:01,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741870_1047 (size=175) 2024-12-02T16:23:01,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741870_1047 (size=175) 2024-12-02T16:23:01,917 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T16:23:01,917 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733156581472) 2024-12-02T16:23:01,933 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=403 (was 399) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:58774 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:54488 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:43664 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=253 (was 253), ProcessCount=11 (was 11), AvailableMemoryMB=6092 (was 6181) 2024-12-02T16:23:01,946 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=403, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=253, ProcessCount=11, AvailableMemoryMB=6091 2024-12-02T16:23:01,962 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:23:01,969 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733156581961, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:01,970 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733156581970 2024-12-02T16:23:02,002 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733156581961/wal.1733156581970 2024-12-02T16:23:02,005 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:35029:35029)] 2024-12-02T16:23:02,007 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f25d5746bd893f6997a0ca391fa8e702, NAME => 'testSequentialEditLogSeqNum,,1733156581962.f25d5746bd893f6997a0ca391fa8e702.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:02,007 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733156581962.f25d5746bd893f6997a0ca391fa8e702.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:02,007 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f25d5746bd893f6997a0ca391fa8e702 2024-12-02T16:23:02,007 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f25d5746bd893f6997a0ca391fa8e702 2024-12-02T16:23:02,008 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:33443/hbase/data/default/testSequentialEditLogSeqNum/f25d5746bd893f6997a0ca391fa8e702 doesn't exist for region: f25d5746bd893f6997a0ca391fa8e702 on table testSequentialEditLogSeqNum 2024-12-02T16:23:02,009 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: f25d5746bd893f6997a0ca391fa8e702 on table testSequentialEditLogSeqNum 2024-12-02T16:23:02,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741875_1052 (size=62) 2024-12-02T16:23:02,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741875_1052 (size=62) 2024-12-02T16:23:02,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741875_1052 (size=62) 2024-12-02T16:23:02,026 INFO [StoreOpener-f25d5746bd893f6997a0ca391fa8e702-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f25d5746bd893f6997a0ca391fa8e702 2024-12-02T16:23:02,028 INFO [StoreOpener-f25d5746bd893f6997a0ca391fa8e702-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f25d5746bd893f6997a0ca391fa8e702 columnFamilyName a 2024-12-02T16:23:02,028 DEBUG [StoreOpener-f25d5746bd893f6997a0ca391fa8e702-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:02,028 INFO [StoreOpener-f25d5746bd893f6997a0ca391fa8e702-1 {}] regionserver.HStore(327): Store=f25d5746bd893f6997a0ca391fa8e702/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:02,029 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f25d5746bd893f6997a0ca391fa8e702 2024-12-02T16:23:02,029 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testSequentialEditLogSeqNum/f25d5746bd893f6997a0ca391fa8e702 2024-12-02T16:23:02,030 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testSequentialEditLogSeqNum/f25d5746bd893f6997a0ca391fa8e702 2024-12-02T16:23:02,030 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f25d5746bd893f6997a0ca391fa8e702 2024-12-02T16:23:02,030 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f25d5746bd893f6997a0ca391fa8e702 2024-12-02T16:23:02,032 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f25d5746bd893f6997a0ca391fa8e702 2024-12-02T16:23:02,035 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testSequentialEditLogSeqNum/f25d5746bd893f6997a0ca391fa8e702/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T16:23:02,035 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f25d5746bd893f6997a0ca391fa8e702; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74022523, jitterRate=0.1030215471982956}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T16:23:02,036 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f25d5746bd893f6997a0ca391fa8e702: Writing region info on filesystem at 1733156582007Initializing all the Stores at 1733156582021 (+14 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156582021Cleaning up temporary data from old regions at 1733156582030 (+9 ms)Region opened successfully at 1733156582036 (+6 ms) 2024-12-02T16:23:02,047 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f25d5746bd893f6997a0ca391fa8e702 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-02T16:23:02,067 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testSequentialEditLogSeqNum/f25d5746bd893f6997a0ca391fa8e702/.tmp/a/3408e849663a47c2bc47afdc780ca9d4 is 81, key is testSequentialEditLogSeqNum/a:x0/1733156582037/Put/seqid=0 2024-12-02T16:23:02,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741876_1053 (size=5833) 2024-12-02T16:23:02,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741876_1053 (size=5833) 2024-12-02T16:23:02,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741876_1053 (size=5833) 2024-12-02T16:23:02,074 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testSequentialEditLogSeqNum/f25d5746bd893f6997a0ca391fa8e702/.tmp/a/3408e849663a47c2bc47afdc780ca9d4 2024-12-02T16:23:02,081 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testSequentialEditLogSeqNum/f25d5746bd893f6997a0ca391fa8e702/.tmp/a/3408e849663a47c2bc47afdc780ca9d4 as hdfs://localhost:33443/hbase/data/default/testSequentialEditLogSeqNum/f25d5746bd893f6997a0ca391fa8e702/a/3408e849663a47c2bc47afdc780ca9d4 2024-12-02T16:23:02,087 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testSequentialEditLogSeqNum/f25d5746bd893f6997a0ca391fa8e702/a/3408e849663a47c2bc47afdc780ca9d4, entries=10, sequenceid=13, filesize=5.7 K 2024-12-02T16:23:02,089 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for f25d5746bd893f6997a0ca391fa8e702 in 42ms, sequenceid=13, compaction requested=false 2024-12-02T16:23:02,089 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f25d5746bd893f6997a0ca391fa8e702: 2024-12-02T16:23:02,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T16:23:02,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T16:23:02,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T16:23:02,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T16:23:02,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T16:23:02,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741874_1051 (size=3256) 2024-12-02T16:23:02,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741874_1051 (size=3256) 2024-12-02T16:23:02,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741874_1051 (size=3256) 2024-12-02T16:23:02,114 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33443/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733156581961/wal.1733156581970, size=3.2 K (3256bytes) 2024-12-02T16:23:02,115 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33443/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733156581961/wal.1733156581970 2024-12-02T16:23:02,115 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33443/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733156581961/wal.1733156581970 after 0ms 2024-12-02T16:23:02,142 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:02,142 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33443/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733156581961/wal.1733156581970 took 28ms 2024-12-02T16:23:02,146 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33443/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733156581961/wal.1733156581970 so closing down 2024-12-02T16:23:02,146 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T16:23:02,147 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733156581970.temp 2024-12-02T16:23:02,173 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testSequentialEditLogSeqNum/f25d5746bd893f6997a0ca391fa8e702/recovered.edits/0000000000000000003-wal.1733156581970.temp 2024-12-02T16:23:02,174 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T16:23:02,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741877_1054 (size=2650) 2024-12-02T16:23:02,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741877_1054 (size=2650) 2024-12-02T16:23:02,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741877_1054 (size=2650) 2024-12-02T16:23:02,183 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testSequentialEditLogSeqNum/f25d5746bd893f6997a0ca391fa8e702/recovered.edits/0000000000000000003-wal.1733156581970.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-02T16:23:02,185 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33443/hbase/data/default/testSequentialEditLogSeqNum/f25d5746bd893f6997a0ca391fa8e702/recovered.edits/0000000000000000003-wal.1733156581970.temp to hdfs://localhost:33443/hbase/data/default/testSequentialEditLogSeqNum/f25d5746bd893f6997a0ca391fa8e702/recovered.edits/0000000000000000020 2024-12-02T16:23:02,185 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 42 ms; skipped=2; WAL=hdfs://localhost:33443/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733156581961/wal.1733156581970, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-12-02T16:23:02,186 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33443/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733156581961/wal.1733156581970, journal: Splitting hdfs://localhost:33443/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733156581961/wal.1733156581970, size=3.2 K (3256bytes) at 1733156582115Finishing writing output for hdfs://localhost:33443/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733156581961/wal.1733156581970 so closing down at 1733156582146 (+31 ms)Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testSequentialEditLogSeqNum/f25d5746bd893f6997a0ca391fa8e702/recovered.edits/0000000000000000003-wal.1733156581970.temp at 1733156582174 (+28 ms)3 split writer threads finished at 1733156582174Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testSequentialEditLogSeqNum/f25d5746bd893f6997a0ca391fa8e702/recovered.edits/0000000000000000003-wal.1733156581970.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733156582183 (+9 ms)Rename recovered edits hdfs://localhost:33443/hbase/data/default/testSequentialEditLogSeqNum/f25d5746bd893f6997a0ca391fa8e702/recovered.edits/0000000000000000003-wal.1733156581970.temp to hdfs://localhost:33443/hbase/data/default/testSequentialEditLogSeqNum/f25d5746bd893f6997a0ca391fa8e702/recovered.edits/0000000000000000020 at 1733156582185 (+2 ms)Processed 17 edits across 1 Regions in 42 ms; skipped=2; WAL=hdfs://localhost:33443/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733156581961/wal.1733156581970, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1733156582185 2024-12-02T16:23:02,200 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=408 (was 403) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:58774 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:54488 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=241 (was 253), ProcessCount=11 (was 11), AvailableMemoryMB=6088 (was 6091) 2024-12-02T16:23:02,214 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=408, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=241, ProcessCount=11, AvailableMemoryMB=6086 2024-12-02T16:23:02,229 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:23:02,232 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:23:02,254 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T16:23:02,257 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23122900, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/hregion-23122900, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:02,270 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23122900/hregion-23122900.1733156582258, exclude list is [], retry=0 2024-12-02T16:23:02,273 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:02,273 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:02,275 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:02,311 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23122900/hregion-23122900.1733156582258 2024-12-02T16:23:02,311 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:33821:33821)] 2024-12-02T16:23:02,311 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4e0c30a88603b8fdbe13c625f79a35ba, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33443/hbase 2024-12-02T16:23:02,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741879_1056 (size=70) 2024-12-02T16:23:02,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741879_1056 (size=70) 2024-12-02T16:23:02,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741879_1056 (size=70) 2024-12-02T16:23:02,320 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:02,323 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,325 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e0c30a88603b8fdbe13c625f79a35ba columnFamilyName a 2024-12-02T16:23:02,325 DEBUG [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:02,326 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(327): Store=4e0c30a88603b8fdbe13c625f79a35ba/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:02,326 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,328 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e0c30a88603b8fdbe13c625f79a35ba columnFamilyName b 2024-12-02T16:23:02,328 DEBUG [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:02,328 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(327): Store=4e0c30a88603b8fdbe13c625f79a35ba/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:02,328 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,330 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e0c30a88603b8fdbe13c625f79a35ba columnFamilyName c 2024-12-02T16:23:02,330 DEBUG [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:02,330 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(327): Store=4e0c30a88603b8fdbe13c625f79a35ba/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:02,331 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,332 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,332 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,334 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,334 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,335 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:02,336 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,339 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T16:23:02,339 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4e0c30a88603b8fdbe13c625f79a35ba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60126966, jitterRate=-0.10403838753700256}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:02,340 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4e0c30a88603b8fdbe13c625f79a35ba: Writing region info on filesystem at 1733156582321Initializing all the Stores at 1733156582322 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156582322Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156582322Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156582323 (+1 ms)Cleaning up temporary data from old regions at 1733156582334 (+11 ms)Region opened successfully at 1733156582340 (+6 ms) 2024-12-02T16:23:02,340 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4e0c30a88603b8fdbe13c625f79a35ba, disabling compactions & flushes 2024-12-02T16:23:02,341 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba. 2024-12-02T16:23:02,341 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba. 2024-12-02T16:23:02,341 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba. after waiting 0 ms 2024-12-02T16:23:02,341 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba. 2024-12-02T16:23:02,341 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba. 2024-12-02T16:23:02,341 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4e0c30a88603b8fdbe13c625f79a35ba: Waiting for close lock at 1733156582340Disabling compacts and flushes for region at 1733156582340Disabling writes for close at 1733156582341 (+1 ms)Writing region close event to WAL at 1733156582341Closed at 1733156582341 2024-12-02T16:23:02,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741878_1055 (size=175) 2024-12-02T16:23:02,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741878_1055 (size=175) 2024-12-02T16:23:02,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741878_1055 (size=175) 2024-12-02T16:23:02,347 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T16:23:02,347 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23122900:(num 1733156582258) 2024-12-02T16:23:02,347 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:23:02,349 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:02,366 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350, exclude list is [], retry=0 2024-12-02T16:23:02,369 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:02,370 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:02,370 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:02,396 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350 2024-12-02T16:23:02,396 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:43815:43815)] 2024-12-02T16:23:02,396 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e0c30a88603b8fdbe13c625f79a35ba, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:02,396 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:02,397 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,397 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,399 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,400 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e0c30a88603b8fdbe13c625f79a35ba columnFamilyName a 2024-12-02T16:23:02,400 DEBUG [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:02,400 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(327): Store=4e0c30a88603b8fdbe13c625f79a35ba/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:02,400 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,401 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e0c30a88603b8fdbe13c625f79a35ba columnFamilyName b 2024-12-02T16:23:02,401 DEBUG [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:02,402 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(327): Store=4e0c30a88603b8fdbe13c625f79a35ba/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:02,402 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,403 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e0c30a88603b8fdbe13c625f79a35ba columnFamilyName c 2024-12-02T16:23:02,403 DEBUG [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:02,403 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(327): Store=4e0c30a88603b8fdbe13c625f79a35ba/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:02,404 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,404 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,406 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,407 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,407 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,408 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:02,409 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:02,410 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4e0c30a88603b8fdbe13c625f79a35ba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60654776, jitterRate=-0.09617340564727783}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:02,410 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4e0c30a88603b8fdbe13c625f79a35ba: Writing region info on filesystem at 1733156582397Initializing all the Stores at 1733156582398 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156582398Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156582398Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156582398Cleaning up temporary data from old regions at 1733156582407 (+9 ms)Region opened successfully at 1733156582410 (+3 ms) 2024-12-02T16:23:02,416 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733156582414/Put/seqid=0 2024-12-02T16:23:02,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741881_1058 (size=4826) 2024-12-02T16:23:02,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741881_1058 (size=4826) 2024-12-02T16:23:02,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741881_1058 (size=4826) 2024-12-02T16:23:02,431 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33443/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 4e0c30a88603b8fdbe13c625f79a35ba/a 2024-12-02T16:23:02,439 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-02T16:23:02,439 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-02T16:23:02,440 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4e0c30a88603b8fdbe13c625f79a35ba: 2024-12-02T16:23:02,441 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/a/523536886ebc4814b5afda12249eb425_SeqId_3_ 2024-12-02T16:23:02,442 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33443/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 4e0c30a88603b8fdbe13c625f79a35ba/a as hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/a/523536886ebc4814b5afda12249eb425_SeqId_3_ - updating store file list. 2024-12-02T16:23:02,448 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 523536886ebc4814b5afda12249eb425_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-02T16:23:02,449 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/a/523536886ebc4814b5afda12249eb425_SeqId_3_ into 4e0c30a88603b8fdbe13c625f79a35ba/a 2024-12-02T16:23:02,449 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33443/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 4e0c30a88603b8fdbe13c625f79a35ba/a (new location: hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/a/523536886ebc4814b5afda12249eb425_SeqId_3_) 2024-12-02T16:23:02,486 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33443/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350, size=0 (0bytes) 2024-12-02T16:23:02,486 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33443/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350 might be still open, length is 0 2024-12-02T16:23:02,486 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33443/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350 2024-12-02T16:23:02,486 WARN [IPC Server handler 4 on default port 33443 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-02T16:23:02,487 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33443/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350 after 1ms 2024-12-02T16:23:04,823 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:43782 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33271:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43782 dst: /127.0.0.1:33271 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33271 remote=/127.0.0.1:43782]. Total timeout mills is 60000, 57632 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:04,825 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:58846 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33041:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58846 dst: /127.0.0.1:33041 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:04,825 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:54738 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38643:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54738 dst: /127.0.0.1:38643 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:04,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741880_1059 (size=606) 2024-12-02T16:23:04,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741880_1059 (size=606) 2024-12-02T16:23:04,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741880_1059 (size=606) 2024-12-02T16:23:06,489 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33443/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350 after 4003ms 2024-12-02T16:23:06,521 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:06,521 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33443/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350 took 4035ms 2024-12-02T16:23:06,523 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33443/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350; continuing. 2024-12-02T16:23:06,523 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33443/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350 so closing down 2024-12-02T16:23:06,523 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T16:23:06,525 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733156582350.temp 2024-12-02T16:23:06,548 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/recovered.edits/0000000000000000005-wal.1733156582350.temp 2024-12-02T16:23:06,548 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T16:23:06,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741882_1060 (size=329) 2024-12-02T16:23:06,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741882_1060 (size=329) 2024-12-02T16:23:06,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741882_1060 (size=329) 2024-12-02T16:23:06,555 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/recovered.edits/0000000000000000005-wal.1733156582350.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-02T16:23:06,557 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/recovered.edits/0000000000000000005-wal.1733156582350.temp to hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/recovered.edits/0000000000000000005 2024-12-02T16:23:06,557 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:33443/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350, size=0, length=0, corrupted=false, cancelled=false 2024-12-02T16:23:06,557 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33443/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350, journal: Splitting hdfs://localhost:33443/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350, size=0 (0bytes) at 1733156582486Finishing writing output for hdfs://localhost:33443/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350 so closing down at 1733156586523 (+4037 ms)Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/recovered.edits/0000000000000000005-wal.1733156582350.temp at 1733156586548 (+25 ms)3 split writer threads finished at 1733156586548Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/recovered.edits/0000000000000000005-wal.1733156582350.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733156586556 (+8 ms)Rename recovered edits hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/recovered.edits/0000000000000000005-wal.1733156582350.temp to hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/recovered.edits/0000000000000000005 at 1733156586557 (+1 ms)Processed 2 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:33443/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350, size=0, length=0, corrupted=false, cancelled=false at 1733156586557 2024-12-02T16:23:06,558 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33443/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350 to hdfs://localhost:33443/hbase/oldWALs/wal.1733156582350 2024-12-02T16:23:06,559 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/recovered.edits/0000000000000000005 2024-12-02T16:23:06,559 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:23:06,562 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:06,576 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156586562, exclude list is [], retry=0 2024-12-02T16:23:06,579 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:06,579 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:06,579 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:06,604 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156586562 2024-12-02T16:23:06,604 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:43815:43815)] 2024-12-02T16:23:06,604 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e0c30a88603b8fdbe13c625f79a35ba, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:06,605 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:06,605 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:06,605 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:06,607 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:06,608 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e0c30a88603b8fdbe13c625f79a35ba columnFamilyName a 2024-12-02T16:23:06,608 DEBUG [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:06,614 DEBUG [StoreFileOpener-4e0c30a88603b8fdbe13c625f79a35ba-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 523536886ebc4814b5afda12249eb425_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-02T16:23:06,614 DEBUG [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/a/523536886ebc4814b5afda12249eb425_SeqId_3_ 2024-12-02T16:23:06,614 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(327): Store=4e0c30a88603b8fdbe13c625f79a35ba/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:06,614 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:06,615 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e0c30a88603b8fdbe13c625f79a35ba columnFamilyName b 2024-12-02T16:23:06,615 DEBUG [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:06,616 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(327): Store=4e0c30a88603b8fdbe13c625f79a35ba/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:06,616 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:06,617 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e0c30a88603b8fdbe13c625f79a35ba columnFamilyName c 2024-12-02T16:23:06,617 DEBUG [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:06,617 INFO [StoreOpener-4e0c30a88603b8fdbe13c625f79a35ba-1 {}] regionserver.HStore(327): Store=4e0c30a88603b8fdbe13c625f79a35ba/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:06,618 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:06,618 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:06,620 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:06,620 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/recovered.edits/0000000000000000005 2024-12-02T16:23:06,644 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:06,645 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/recovered.edits/0000000000000000005 2024-12-02T16:23:06,646 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4e0c30a88603b8fdbe13c625f79a35ba 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-02T16:23:06,659 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/.tmp/a/403a613c107e40c8954a0192b288232f is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733156582453/Put/seqid=0 2024-12-02T16:23:06,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741884_1062 (size=5149) 2024-12-02T16:23:06,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741884_1062 (size=5149) 2024-12-02T16:23:06,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741884_1062 (size=5149) 2024-12-02T16:23:06,665 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/.tmp/a/403a613c107e40c8954a0192b288232f 2024-12-02T16:23:06,672 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/.tmp/a/403a613c107e40c8954a0192b288232f as hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/a/403a613c107e40c8954a0192b288232f 2024-12-02T16:23:06,678 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/a/403a613c107e40c8954a0192b288232f, entries=1, sequenceid=5, filesize=5.0 K 2024-12-02T16:23:06,678 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 4e0c30a88603b8fdbe13c625f79a35ba in 33ms, sequenceid=5, compaction requested=false; wal=null 2024-12-02T16:23:06,679 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/recovered.edits/0000000000000000005 2024-12-02T16:23:06,680 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:06,681 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:06,681 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:06,683 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4e0c30a88603b8fdbe13c625f79a35ba 2024-12-02T16:23:06,685 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4e0c30a88603b8fdbe13c625f79a35ba/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-02T16:23:06,686 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4e0c30a88603b8fdbe13c625f79a35ba; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75456351, jitterRate=0.12438724935054779}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:06,686 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4e0c30a88603b8fdbe13c625f79a35ba: Writing region info on filesystem at 1733156586605Initializing all the Stores at 1733156586606 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156586606Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156586606Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156586606Obtaining lock to block concurrent updates at 1733156586646 (+40 ms)Preparing flush snapshotting stores in 4e0c30a88603b8fdbe13c625f79a35ba at 1733156586646Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733156586646Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba. at 1733156586646Flushing 4e0c30a88603b8fdbe13c625f79a35ba/a: creating writer at 1733156586646Flushing 4e0c30a88603b8fdbe13c625f79a35ba/a: appending metadata at 1733156586658 (+12 ms)Flushing 4e0c30a88603b8fdbe13c625f79a35ba/a: closing flushed file at 1733156586658Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@734eeecc: reopening flushed file at 1733156586671 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 4e0c30a88603b8fdbe13c625f79a35ba in 33ms, sequenceid=5, compaction requested=false; wal=null at 1733156586678 (+7 ms)Cleaning up temporary data from old regions at 1733156586681 (+3 ms)Region opened successfully at 1733156586686 (+5 ms) 2024-12-02T16:23:06,690 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4e0c30a88603b8fdbe13c625f79a35ba, disabling compactions & flushes 2024-12-02T16:23:06,690 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba. 2024-12-02T16:23:06,690 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba. 2024-12-02T16:23:06,690 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba. after waiting 0 ms 2024-12-02T16:23:06,690 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba. 2024-12-02T16:23:06,691 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733156582230.4e0c30a88603b8fdbe13c625f79a35ba. 2024-12-02T16:23:06,691 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4e0c30a88603b8fdbe13c625f79a35ba: Waiting for close lock at 1733156586690Disabling compacts and flushes for region at 1733156586690Disabling writes for close at 1733156586690Writing region close event to WAL at 1733156586691 (+1 ms)Closed at 1733156586691 2024-12-02T16:23:06,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741883_1061 (size=175) 2024-12-02T16:23:06,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741883_1061 (size=175) 2024-12-02T16:23:06,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741883_1061 (size=175) 2024-12-02T16:23:06,697 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T16:23:06,697 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733156586562) 2024-12-02T16:23:06,711 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=410 (was 408) Potentially hanging thread: IPC Client (1685805399) connection to localhost/127.0.0.1:33443 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-135359009_22 at /127.0.0.1:37022 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-135359009_22 at /127.0.0.1:36284 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:33443 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1022 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=241 (was 241), ProcessCount=11 (was 11), AvailableMemoryMB=6078 (was 6086) 2024-12-02T16:23:06,722 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=410, OpenFileDescriptor=1022, MaxFileDescriptor=1048576, SystemLoadAverage=241, ProcessCount=11, AvailableMemoryMB=6077 2024-12-02T16:23:06,737 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:23:06,741 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T16:23:06,745 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e5d35c688fae,38905,1733156570684 2024-12-02T16:23:06,747 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@265a1a0e 2024-12-02T16:23:06,748 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T16:23:06,750 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T16:23:06,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T16:23:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-02T16:23:06,762 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T16:23:06,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-02T16:23:06,765 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:06,767 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T16:23:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T16:23:06,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741885_1063 (size=694) 2024-12-02T16:23:06,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741885_1063 (size=694) 2024-12-02T16:23:06,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741885_1063 (size=694) 2024-12-02T16:23:06,780 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 01e69ba503cdc68fb29228453655d981, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b 2024-12-02T16:23:06,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741886_1064 (size=77) 2024-12-02T16:23:06,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741886_1064 (size=77) 2024-12-02T16:23:06,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741886_1064 (size=77) 2024-12-02T16:23:06,791 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:06,791 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 01e69ba503cdc68fb29228453655d981, disabling compactions & flushes 2024-12-02T16:23:06,791 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:06,791 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:06,791 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. after waiting 0 ms 2024-12-02T16:23:06,791 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:06,791 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:06,791 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 01e69ba503cdc68fb29228453655d981: Waiting for close lock at 1733156586791Disabling compacts and flushes for region at 1733156586791Disabling writes for close at 1733156586791Writing region close event to WAL at 1733156586791Closed at 1733156586791 2024-12-02T16:23:06,793 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T16:23:06,799 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733156586793"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733156586793"}]},"ts":"1733156586793"} 2024-12-02T16:23:06,803 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T16:23:06,805 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T16:23:06,809 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733156586806"}]},"ts":"1733156586806"} 2024-12-02T16:23:06,817 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-02T16:23:06,817 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {e5d35c688fae=0} racks are {/default-rack=0} 2024-12-02T16:23:06,819 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T16:23:06,819 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T16:23:06,819 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-02T16:23:06,819 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T16:23:06,819 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T16:23:06,819 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-02T16:23:06,819 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T16:23:06,819 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T16:23:06,819 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-02T16:23:06,819 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T16:23:06,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, ASSIGN}] 2024-12-02T16:23:06,823 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, ASSIGN 2024-12-02T16:23:06,824 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, ASSIGN; state=OFFLINE, location=e5d35c688fae,36399,1733156571440; forceNewPlan=false, retain=false 2024-12-02T16:23:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T16:23:06,980 INFO [e5d35c688fae:38905 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-02T16:23:06,982 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=01e69ba503cdc68fb29228453655d981, regionState=OPENING, regionLocation=e5d35c688fae,36399,1733156571440 2024-12-02T16:23:06,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, ASSIGN because future has completed 2024-12-02T16:23:06,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,36399,1733156571440}] 2024-12-02T16:23:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T16:23:07,154 INFO [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:07,155 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 01e69ba503cdc68fb29228453655d981, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:07,156 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:07,156 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:07,156 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:07,156 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:07,159 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:07,161 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01e69ba503cdc68fb29228453655d981 columnFamilyName cf1 2024-12-02T16:23:07,161 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:07,162 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(327): Store=01e69ba503cdc68fb29228453655d981/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:07,162 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:07,164 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01e69ba503cdc68fb29228453655d981 columnFamilyName cf2 2024-12-02T16:23:07,165 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:07,165 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(327): Store=01e69ba503cdc68fb29228453655d981/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:07,166 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:07,167 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:07,168 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:07,169 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:07,169 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:07,170 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-02T16:23:07,172 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:07,176 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T16:23:07,177 INFO [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 01e69ba503cdc68fb29228453655d981; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72248780, jitterRate=0.07659071683883667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-02T16:23:07,177 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:07,178 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 01e69ba503cdc68fb29228453655d981: Running coprocessor pre-open hook at 1733156587157Writing region info on filesystem at 1733156587157Initializing all the Stores at 1733156587158 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156587158Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156587159 (+1 ms)Cleaning up temporary data from old regions at 1733156587169 (+10 ms)Running coprocessor post-open hooks at 1733156587177 (+8 ms)Region opened successfully at 1733156587178 (+1 ms) 2024-12-02T16:23:07,180 INFO [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., pid=6, masterSystemTime=1733156587142 2024-12-02T16:23:07,183 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:07,183 INFO [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:07,184 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=01e69ba503cdc68fb29228453655d981, regionState=OPEN, openSeqNum=2, regionLocation=e5d35c688fae,36399,1733156571440 2024-12-02T16:23:07,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,36399,1733156571440 because future has completed 2024-12-02T16:23:07,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-02T16:23:07,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,36399,1733156571440 in 202 msec 2024-12-02T16:23:07,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-02T16:23:07,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, ASSIGN in 374 msec 2024-12-02T16:23:07,198 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T16:23:07,199 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733156587198"}]},"ts":"1733156587198"} 2024-12-02T16:23:07,201 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-02T16:23:07,202 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T16:23:07,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 448 msec 2024-12-02T16:23:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T16:23:07,399 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-02T16:23:07,399 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-02T16:23:07,402 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T16:23:07,412 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-02T16:23:07,412 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T16:23:07,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-02T16:23:07,429 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., hostname=e5d35c688fae,36399,1733156571440, seqNum=2] 2024-12-02T16:23:07,444 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=01e69ba503cdc68fb29228453655d981, source=e5d35c688fae,36399,1733156571440, destination=e5d35c688fae,35297,1733156571540, warming up region on e5d35c688fae,35297,1733156571540 2024-12-02T16:23:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T16:23:07,447 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=01e69ba503cdc68fb29228453655d981, source=e5d35c688fae,36399,1733156571440, destination=e5d35c688fae,35297,1733156571540, running balancer 2024-12-02T16:23:07,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51103, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T16:23:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, REOPEN/MOVE 2024-12-02T16:23:07,449 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, REOPEN/MOVE 2024-12-02T16:23:07,452 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=01e69ba503cdc68fb29228453655d981, regionState=CLOSING, regionLocation=e5d35c688fae,36399,1733156571440 2024-12-02T16:23:07,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35297 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35297 {}] regionserver.HRegion(7855): Warmup {ENCODED => 01e69ba503cdc68fb29228453655d981, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35297 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:07,455 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:07,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, REOPEN/MOVE because future has completed 2024-12-02T16:23:07,457 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01e69ba503cdc68fb29228453655d981 columnFamilyName cf1 2024-12-02T16:23:07,457 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:07,457 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T16:23:07,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,36399,1733156571440}] 2024-12-02T16:23:07,458 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(327): Store=01e69ba503cdc68fb29228453655d981/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:07,458 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:07,460 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01e69ba503cdc68fb29228453655d981 columnFamilyName cf2 2024-12-02T16:23:07,460 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:07,461 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(327): Store=01e69ba503cdc68fb29228453655d981/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35297 {}] regionserver.HRegion(1722): Closing 01e69ba503cdc68fb29228453655d981, disabling compactions & flushes 2024-12-02T16:23:07,461 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35297 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35297 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35297 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. after waiting 0 ms 2024-12-02T16:23:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35297 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:07,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35297 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35297 {}] regionserver.HRegion(1676): Region close journal for 01e69ba503cdc68fb29228453655d981: Waiting for close lock at 1733156587461Disabling compacts and flushes for region at 1733156587461Disabling writes for close at 1733156587461Writing region close event to WAL at 1733156587469 (+8 ms)Closed at 1733156587469 2024-12-02T16:23:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-02T16:23:07,618 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:07,618 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T16:23:07,619 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 01e69ba503cdc68fb29228453655d981, disabling compactions & flushes 2024-12-02T16:23:07,619 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:07,619 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:07,619 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. after waiting 0 ms 2024-12-02T16:23:07,619 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:07,619 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 01e69ba503cdc68fb29228453655d981 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-02T16:23:07,647 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/.tmp/cf1/e6ffb4e1e889486091a01f9d19a9cae2 is 35, key is r1/cf1:q/1733156587431/Put/seqid=0 2024-12-02T16:23:07,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741887_1065 (size=4783) 2024-12-02T16:23:07,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741887_1065 (size=4783) 2024-12-02T16:23:07,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741887_1065 (size=4783) 2024-12-02T16:23:07,656 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/.tmp/cf1/e6ffb4e1e889486091a01f9d19a9cae2 2024-12-02T16:23:07,664 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/.tmp/cf1/e6ffb4e1e889486091a01f9d19a9cae2 as hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/e6ffb4e1e889486091a01f9d19a9cae2 2024-12-02T16:23:07,671 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/e6ffb4e1e889486091a01f9d19a9cae2, entries=1, sequenceid=5, filesize=4.7 K 2024-12-02T16:23:07,672 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 01e69ba503cdc68fb29228453655d981 in 53ms, sequenceid=5, compaction requested=false 2024-12-02T16:23:07,673 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-02T16:23:07,678 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-02T16:23:07,681 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:07,681 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 01e69ba503cdc68fb29228453655d981: Waiting for close lock at 1733156587618Running coprocessor pre-close hooks at 1733156587618Disabling compacts and flushes for region at 1733156587618Disabling writes for close at 1733156587619 (+1 ms)Obtaining lock to block concurrent updates at 1733156587619Preparing flush snapshotting stores in 01e69ba503cdc68fb29228453655d981 at 1733156587619Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733156587619Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. at 1733156587620 (+1 ms)Flushing 01e69ba503cdc68fb29228453655d981/cf1: creating writer at 1733156587621 (+1 ms)Flushing 01e69ba503cdc68fb29228453655d981/cf1: appending metadata at 1733156587646 (+25 ms)Flushing 01e69ba503cdc68fb29228453655d981/cf1: closing flushed file at 1733156587646Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78203ee6: reopening flushed file at 1733156587663 (+17 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 01e69ba503cdc68fb29228453655d981 in 53ms, sequenceid=5, compaction requested=false at 1733156587673 (+10 ms)Writing region close event to WAL at 1733156587674 (+1 ms)Running coprocessor post-close hooks at 1733156587679 (+5 ms)Closed at 1733156587680 (+1 ms) 2024-12-02T16:23:07,681 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 01e69ba503cdc68fb29228453655d981 move to e5d35c688fae,35297,1733156571540 record at close sequenceid=5 2024-12-02T16:23:07,684 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:07,685 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=01e69ba503cdc68fb29228453655d981, regionState=CLOSED 2024-12-02T16:23:07,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,36399,1733156571440 because future has completed 2024-12-02T16:23:07,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-02T16:23:07,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,36399,1733156571440 in 233 msec 2024-12-02T16:23:07,694 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, REOPEN/MOVE; state=CLOSED, location=e5d35c688fae,35297,1733156571540; forceNewPlan=false, retain=false 2024-12-02T16:23:07,844 INFO [e5d35c688fae:38905 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-02T16:23:07,845 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=01e69ba503cdc68fb29228453655d981, regionState=OPENING, regionLocation=e5d35c688fae,35297,1733156571540 2024-12-02T16:23:07,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, REOPEN/MOVE because future has completed 2024-12-02T16:23:07,848 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,35297,1733156571540}] 2024-12-02T16:23:08,006 INFO [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:08,006 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 01e69ba503cdc68fb29228453655d981, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:08,007 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,007 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:08,007 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,007 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,008 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,009 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01e69ba503cdc68fb29228453655d981 columnFamilyName cf1 2024-12-02T16:23:08,010 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:08,017 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/e6ffb4e1e889486091a01f9d19a9cae2 2024-12-02T16:23:08,017 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(327): Store=01e69ba503cdc68fb29228453655d981/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:08,017 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,018 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01e69ba503cdc68fb29228453655d981 columnFamilyName cf2 2024-12-02T16:23:08,018 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:08,019 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(327): Store=01e69ba503cdc68fb29228453655d981/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:08,019 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,020 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,021 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,022 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,022 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,023 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-02T16:23:08,024 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,025 INFO [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 01e69ba503cdc68fb29228453655d981; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65468069, jitterRate=-0.024449750781059265}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-02T16:23:08,025 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,026 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 01e69ba503cdc68fb29228453655d981: Running coprocessor pre-open hook at 1733156588007Writing region info on filesystem at 1733156588007Initializing all the Stores at 1733156588008 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156588008Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156588008Cleaning up temporary data from old regions at 1733156588022 (+14 ms)Running coprocessor post-open hooks at 1733156588025 (+3 ms)Region opened successfully at 1733156588026 (+1 ms) 2024-12-02T16:23:08,027 INFO [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., pid=9, masterSystemTime=1733156588001 2024-12-02T16:23:08,030 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:08,030 INFO [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:08,031 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=01e69ba503cdc68fb29228453655d981, regionState=OPEN, openSeqNum=9, regionLocation=e5d35c688fae,35297,1733156571540 2024-12-02T16:23:08,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,35297,1733156571540 because future has completed 2024-12-02T16:23:08,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-02T16:23:08,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,35297,1733156571540 in 188 msec 2024-12-02T16:23:08,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, REOPEN/MOVE in 593 msec 2024-12-02T16:23:08,055 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T16:23:08,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T16:23:08,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36399 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:56560 deadline: 1733156648061, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e5d35c688fae port=35297 startCode=1733156571540. As of locationSeqNum=5. 2024-12-02T16:23:08,089 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., hostname=e5d35c688fae,36399,1733156571440, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., hostname=e5d35c688fae,36399,1733156571440, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e5d35c688fae port=35297 startCode=1733156571540. As of locationSeqNum=5. 2024-12-02T16:23:08,090 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., hostname=e5d35c688fae,36399,1733156571440, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e5d35c688fae port=35297 startCode=1733156571540. As of locationSeqNum=5. 2024-12-02T16:23:08,090 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., hostname=e5d35c688fae,36399,1733156571440, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., hostname=e5d35c688fae,35297,1733156571540, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e5d35c688fae port=35297 startCode=1733156571540. As of locationSeqNum=5. 2024-12-02T16:23:08,196 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T16:23:08,199 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T16:23:08,208 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 01e69ba503cdc68fb29228453655d981 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-02T16:23:08,223 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/.tmp/cf1/c5abbe506ef14e04a68f94f115be2581 is 29, key is r1/cf1:/1733156588200/DeleteFamily/seqid=0 2024-12-02T16:23:08,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741888_1066 (size=4906) 2024-12-02T16:23:08,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741888_1066 (size=4906) 2024-12-02T16:23:08,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741888_1066 (size=4906) 2024-12-02T16:23:08,233 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/.tmp/cf1/c5abbe506ef14e04a68f94f115be2581 2024-12-02T16:23:08,240 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c5abbe506ef14e04a68f94f115be2581 2024-12-02T16:23:08,255 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/.tmp/cf2/298c0ca4873b4addaaa5a2230fc857a5 is 29, key is r1/cf2:/1733156588200/DeleteFamily/seqid=0 2024-12-02T16:23:08,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741889_1067 (size=4906) 2024-12-02T16:23:08,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741889_1067 (size=4906) 2024-12-02T16:23:08,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741889_1067 (size=4906) 2024-12-02T16:23:08,263 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/.tmp/cf2/298c0ca4873b4addaaa5a2230fc857a5 2024-12-02T16:23:08,271 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 298c0ca4873b4addaaa5a2230fc857a5 2024-12-02T16:23:08,272 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/.tmp/cf1/c5abbe506ef14e04a68f94f115be2581 as hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/c5abbe506ef14e04a68f94f115be2581 2024-12-02T16:23:08,279 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c5abbe506ef14e04a68f94f115be2581 2024-12-02T16:23:08,279 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/c5abbe506ef14e04a68f94f115be2581, entries=1, sequenceid=12, filesize=4.8 K 2024-12-02T16:23:08,280 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/.tmp/cf2/298c0ca4873b4addaaa5a2230fc857a5 as hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf2/298c0ca4873b4addaaa5a2230fc857a5 2024-12-02T16:23:08,287 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 298c0ca4873b4addaaa5a2230fc857a5 2024-12-02T16:23:08,287 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf2/298c0ca4873b4addaaa5a2230fc857a5, entries=1, sequenceid=12, filesize=4.8 K 2024-12-02T16:23:08,288 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 01e69ba503cdc68fb29228453655d981 in 80ms, sequenceid=12, compaction requested=false 2024-12-02T16:23:08,288 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 01e69ba503cdc68fb29228453655d981: 2024-12-02T16:23:08,291 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T16:23:08,292 DEBUG [Time-limited test {}] regionserver.HStore(1541): 01e69ba503cdc68fb29228453655d981/cf1 is initiating major compaction (all files) 2024-12-02T16:23:08,293 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T16:23:08,293 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T16:23:08,294 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 01e69ba503cdc68fb29228453655d981/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:08,294 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/e6ffb4e1e889486091a01f9d19a9cae2, hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/c5abbe506ef14e04a68f94f115be2581] into tmpdir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/.tmp, totalSize=9.5 K 2024-12-02T16:23:08,295 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e6ffb4e1e889486091a01f9d19a9cae2, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733156587431 2024-12-02T16:23:08,296 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c5abbe506ef14e04a68f94f115be2581, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-02T16:23:08,308 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 01e69ba503cdc68fb29228453655d981#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T16:23:08,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741890_1068 (size=4626) 2024-12-02T16:23:08,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741890_1068 (size=4626) 2024-12-02T16:23:08,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741890_1068 (size=4626) 2024-12-02T16:23:08,328 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/.tmp/cf1/9351bcb6240441149880e3136a6424ca as hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/9351bcb6240441149880e3136a6424ca 2024-12-02T16:23:08,344 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 01e69ba503cdc68fb29228453655d981/cf1 of 01e69ba503cdc68fb29228453655d981 into 9351bcb6240441149880e3136a6424ca(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T16:23:08,344 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 01e69ba503cdc68fb29228453655d981: 2024-12-02T16:23:08,344 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-02T16:23:08,344 DEBUG [Time-limited test {}] regionserver.HStore(1541): 01e69ba503cdc68fb29228453655d981/cf2 is initiating major compaction (all files) 2024-12-02T16:23:08,344 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T16:23:08,344 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T16:23:08,344 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 01e69ba503cdc68fb29228453655d981/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:08,345 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf2/298c0ca4873b4addaaa5a2230fc857a5] into tmpdir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/.tmp, totalSize=4.8 K 2024-12-02T16:23:08,345 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 298c0ca4873b4addaaa5a2230fc857a5, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-02T16:23:08,352 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 01e69ba503cdc68fb29228453655d981#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T16:23:08,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741891_1069 (size=4592) 2024-12-02T16:23:08,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741891_1069 (size=4592) 2024-12-02T16:23:08,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741891_1069 (size=4592) 2024-12-02T16:23:08,367 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/.tmp/cf2/188e2968f4a14a279d18ec5e26d7bff9 as hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf2/188e2968f4a14a279d18ec5e26d7bff9 2024-12-02T16:23:08,376 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 01e69ba503cdc68fb29228453655d981/cf2 of 01e69ba503cdc68fb29228453655d981 into 188e2968f4a14a279d18ec5e26d7bff9(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T16:23:08,376 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 01e69ba503cdc68fb29228453655d981: 2024-12-02T16:23:08,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=01e69ba503cdc68fb29228453655d981, source=e5d35c688fae,35297,1733156571540, destination=e5d35c688fae,36399,1733156571440, warming up region on e5d35c688fae,36399,1733156571440 2024-12-02T16:23:08,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=01e69ba503cdc68fb29228453655d981, source=e5d35c688fae,35297,1733156571540, destination=e5d35c688fae,36399,1733156571440, running balancer 2024-12-02T16:23:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, REOPEN/MOVE 2024-12-02T16:23:08,384 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, REOPEN/MOVE 2024-12-02T16:23:08,386 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=01e69ba503cdc68fb29228453655d981, regionState=CLOSING, regionLocation=e5d35c688fae,35297,1733156571540 2024-12-02T16:23:08,386 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36399 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36399 {}] regionserver.HRegion(7855): Warmup {ENCODED => 01e69ba503cdc68fb29228453655d981, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36399 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:08,387 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,388 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01e69ba503cdc68fb29228453655d981 columnFamilyName cf1 2024-12-02T16:23:08,389 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:08,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, REOPEN/MOVE because future has completed 2024-12-02T16:23:08,389 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T16:23:08,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,35297,1733156571540}] 2024-12-02T16:23:08,398 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/9351bcb6240441149880e3136a6424ca 2024-12-02T16:23:08,405 INFO [StoreFileOpener-01e69ba503cdc68fb29228453655d981-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c5abbe506ef14e04a68f94f115be2581 2024-12-02T16:23:08,405 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/c5abbe506ef14e04a68f94f115be2581 2024-12-02T16:23:08,412 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/e6ffb4e1e889486091a01f9d19a9cae2 2024-12-02T16:23:08,412 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(327): Store=01e69ba503cdc68fb29228453655d981/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:08,412 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,414 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01e69ba503cdc68fb29228453655d981 columnFamilyName cf2 2024-12-02T16:23:08,414 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:08,422 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf2/188e2968f4a14a279d18ec5e26d7bff9 2024-12-02T16:23:08,430 INFO [StoreFileOpener-01e69ba503cdc68fb29228453655d981-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 298c0ca4873b4addaaa5a2230fc857a5 2024-12-02T16:23:08,430 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf2/298c0ca4873b4addaaa5a2230fc857a5 2024-12-02T16:23:08,431 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(327): Store=01e69ba503cdc68fb29228453655d981/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36399 {}] regionserver.HRegion(1722): Closing 01e69ba503cdc68fb29228453655d981, disabling compactions & flushes 2024-12-02T16:23:08,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36399 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36399 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36399 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. after waiting 0 ms 2024-12-02T16:23:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36399 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:08,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36399 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36399 {}] regionserver.HRegion(1676): Region close journal for 01e69ba503cdc68fb29228453655d981: Waiting for close lock at 1733156588431Disabling compacts and flushes for region at 1733156588431Disabling writes for close at 1733156588431Writing region close event to WAL at 1733156588433 (+2 ms)Closed at 1733156588433 2024-12-02T16:23:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-02T16:23:08,544 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,544 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T16:23:08,547 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 01e69ba503cdc68fb29228453655d981, disabling compactions & flushes 2024-12-02T16:23:08,547 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:08,547 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:08,547 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. after waiting 0 ms 2024-12-02T16:23:08,547 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:08,548 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/e6ffb4e1e889486091a01f9d19a9cae2, hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/c5abbe506ef14e04a68f94f115be2581] to archive 2024-12-02T16:23:08,553 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T16:23:08,559 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/e6ffb4e1e889486091a01f9d19a9cae2 to hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/e6ffb4e1e889486091a01f9d19a9cae2 2024-12-02T16:23:08,561 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/c5abbe506ef14e04a68f94f115be2581 to hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/c5abbe506ef14e04a68f94f115be2581 2024-12-02T16:23:08,573 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf2/298c0ca4873b4addaaa5a2230fc857a5] to archive 2024-12-02T16:23:08,574 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T16:23:08,576 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf2/298c0ca4873b4addaaa5a2230fc857a5 to hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf2/298c0ca4873b4addaaa5a2230fc857a5 2024-12-02T16:23:08,581 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-02T16:23:08,582 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:08,582 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 01e69ba503cdc68fb29228453655d981: Waiting for close lock at 1733156588547Running coprocessor pre-close hooks at 1733156588547Disabling compacts and flushes for region at 1733156588547Disabling writes for close at 1733156588547Writing region close event to WAL at 1733156588577 (+30 ms)Running coprocessor post-close hooks at 1733156588582 (+5 ms)Closed at 1733156588582 2024-12-02T16:23:08,582 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 01e69ba503cdc68fb29228453655d981 move to e5d35c688fae,36399,1733156571440 record at close sequenceid=12 2024-12-02T16:23:08,584 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,585 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=01e69ba503cdc68fb29228453655d981, regionState=CLOSED 2024-12-02T16:23:08,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,35297,1733156571540 because future has completed 2024-12-02T16:23:08,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-02T16:23:08,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,35297,1733156571540 in 200 msec 2024-12-02T16:23:08,593 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, REOPEN/MOVE; state=CLOSED, location=e5d35c688fae,36399,1733156571440; forceNewPlan=false, retain=false 2024-12-02T16:23:08,743 INFO [e5d35c688fae:38905 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-02T16:23:08,744 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=01e69ba503cdc68fb29228453655d981, regionState=OPENING, regionLocation=e5d35c688fae,36399,1733156571440 2024-12-02T16:23:08,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, REOPEN/MOVE because future has completed 2024-12-02T16:23:08,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,36399,1733156571440}] 2024-12-02T16:23:08,910 INFO [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:08,910 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 01e69ba503cdc68fb29228453655d981, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:08,910 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,911 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:08,911 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,911 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,914 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,915 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01e69ba503cdc68fb29228453655d981 columnFamilyName cf1 2024-12-02T16:23:08,916 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:08,924 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/9351bcb6240441149880e3136a6424ca 2024-12-02T16:23:08,924 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(327): Store=01e69ba503cdc68fb29228453655d981/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:08,924 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,926 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01e69ba503cdc68fb29228453655d981 columnFamilyName cf2 2024-12-02T16:23:08,926 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:08,934 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf2/188e2968f4a14a279d18ec5e26d7bff9 2024-12-02T16:23:08,934 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(327): Store=01e69ba503cdc68fb29228453655d981/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:08,935 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,936 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,937 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,939 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,939 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,940 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-02T16:23:08,942 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,943 INFO [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 01e69ba503cdc68fb29228453655d981; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69222248, jitterRate=0.031491875648498535}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-02T16:23:08,943 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:08,944 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 01e69ba503cdc68fb29228453655d981: Running coprocessor pre-open hook at 1733156588911Writing region info on filesystem at 1733156588911Initializing all the Stores at 1733156588913 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156588913Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156588913Cleaning up temporary data from old regions at 1733156588939 (+26 ms)Running coprocessor post-open hooks at 1733156588943 (+4 ms)Region opened successfully at 1733156588944 (+1 ms) 2024-12-02T16:23:08,945 INFO [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., pid=12, masterSystemTime=1733156588902 2024-12-02T16:23:08,949 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:08,949 INFO [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:08,950 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=01e69ba503cdc68fb29228453655d981, regionState=OPEN, openSeqNum=18, regionLocation=e5d35c688fae,36399,1733156571440 2024-12-02T16:23:08,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,36399,1733156571440 because future has completed 2024-12-02T16:23:08,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-02T16:23:08,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,36399,1733156571440 in 205 msec 2024-12-02T16:23:08,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, REOPEN/MOVE in 574 msec 2024-12-02T16:23:08,986 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T16:23:08,988 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T16:23:08,989 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server e5d35c688fae,36399,1733156571440: testing ***** 2024-12-02T16:23:08,989 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-02T16:23:08,991 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-02T16:23:08,992 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-02T16:23:08,995 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-02T16:23:08,996 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-02T16:23:09,003 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 630255616 }, "NonHeapMemoryUsage": { "committed": 171900928, "init": 7667712, "max": -1, "used": 169360264 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "e5d35c688fae", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 10, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 9, "ProcessCallTime_95th_percentile": 9, "ProcessCallTime_98th_percentile": 9, "ProcessCallTime_99th_percentile": 9, "ProcessCallTime_99.9th_percentile": 9, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 10, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 9, "TotalCallTime_95th_percentile": 9, "TotalCallTime_98th_percentile": 9, "TotalCallTime_99th_percentile": 9, "TotalCallTime_99.9th_percentile": 9, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "e5d35c688fae", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:61407", "tag.serverName": "e5d35c688fae,43901,1733156571586", "tag.clusterId": "f5805b21-ffb2-47ac-9b04-2a283bcdbef5", "tag.Context": "regionserver", "tag.Hostname": "e5d35c688fae", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733156571586, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-02T16:23:09,007 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38905 {}] master.MasterRpcServices(700): e5d35c688fae,36399,1733156571440 reported a fatal error: ***** ABORTING region server e5d35c688fae,36399,1733156571440: testing ***** 2024-12-02T16:23:09,009 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e5d35c688fae,36399,1733156571440' ***** 2024-12-02T16:23:09,009 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-02T16:23:09,009 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T16:23:09,010 INFO [RS:0;e5d35c688fae:36399 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-02T16:23:09,010 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T16:23:09,010 INFO [RS:0;e5d35c688fae:36399 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-02T16:23:09,010 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.HRegionServer(3091): Received CLOSE for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:09,010 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.HRegionServer(956): aborting server e5d35c688fae,36399,1733156571440 2024-12-02T16:23:09,010 INFO [RS:0;e5d35c688fae:36399 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T16:23:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35297 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:44614 deadline: 1733156649010, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e5d35c688fae port=36399 startCode=1733156571440. As of locationSeqNum=12. 2024-12-02T16:23:09,011 INFO [RS:0;e5d35c688fae:36399 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e5d35c688fae:36399. 2024-12-02T16:23:09,011 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 01e69ba503cdc68fb29228453655d981, disabling compactions & flushes 2024-12-02T16:23:09,011 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:09,011 DEBUG [RS:0;e5d35c688fae:36399 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T16:23:09,011 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:09,011 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. after waiting 0 ms 2024-12-02T16:23:09,011 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:09,011 DEBUG [RS:0;e5d35c688fae:36399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T16:23:09,012 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., hostname=e5d35c688fae,35297,1733156571540, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., hostname=e5d35c688fae,35297,1733156571540, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e5d35c688fae port=36399 startCode=1733156571440. As of locationSeqNum=12. 2024-12-02T16:23:09,012 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., hostname=e5d35c688fae,35297,1733156571540, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e5d35c688fae port=36399 startCode=1733156571440. As of locationSeqNum=12. 2024-12-02T16:23:09,012 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T16:23:09,012 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., hostname=e5d35c688fae,35297,1733156571540, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., hostname=e5d35c688fae,36399,1733156571440, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e5d35c688fae port=36399 startCode=1733156571440. As of locationSeqNum=12. 2024-12-02T16:23:09,012 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T16:23:09,012 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T16:23:09,012 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T16:23:09,013 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-02T16:23:09,013 INFO [regionserver/e5d35c688fae:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T16:23:09,013 DEBUG [RS:0;e5d35c688fae:36399 {}] regionserver.HRegionServer(1325): Online Regions={01e69ba503cdc68fb29228453655d981=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., 1588230740=hbase:meta,,1.1588230740} 2024-12-02T16:23:09,013 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T16:23:09,013 INFO [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T16:23:09,013 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T16:23:09,013 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T16:23:09,013 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T16:23:09,013 DEBUG [RS:0;e5d35c688fae:36399 {}] regionserver.HRegionServer(1351): Waiting on 01e69ba503cdc68fb29228453655d981, 1588230740 2024-12-02T16:23:09,014 ERROR [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-12-02T16:23:09,014 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T16:23:09,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e5d35c688fae,36399,1733156571440 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:09,014 INFO [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T16:23:09,014 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:09,014 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733156589013Running coprocessor pre-close hooks at 1733156589013Disabling compacts and flushes for region at 1733156589013Disabling writes for close at 1733156589013Writing region close event to WAL at 1733156589014 (+1 ms)Running coprocessor post-close hooks at 1733156589014Closed at 1733156589014 2024-12-02T16:23:09,014 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 01e69ba503cdc68fb29228453655d981: Waiting for close lock at 1733156589010Running coprocessor pre-close hooks at 1733156589011 (+1 ms)Disabling compacts and flushes for region at 1733156589011Disabling writes for close at 1733156589011Writing region close event to WAL at 1733156589013 (+2 ms)Running coprocessor post-close hooks at 1733156589013Closed at 1733156589014 (+1 ms) 2024-12-02T16:23:09,014 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T16:23:09,014 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:09,015 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e5d35c688fae,36399,1733156571440 aborting 2024-12-02T16:23:09,015 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e5d35c688fae,36399,1733156571440 aborting 2024-12-02T16:23:09,015 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1 from cache 2024-12-02T16:23:09,126 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T16:23:09,129 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1] 2024-12-02T16:23:09,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e5d35c688fae,36399,1733156571440 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:09,131 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e5d35c688fae,36399,1733156571440 aborting 2024-12-02T16:23:09,131 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e5d35c688fae,36399,1733156571440 aborting 2024-12-02T16:23:09,132 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1 from cache 2024-12-02T16:23:09,214 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.HRegionServer(976): stopping server e5d35c688fae,36399,1733156571440; all regions closed. 2024-12-02T16:23:09,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741836_1012 (size=7323) 2024-12-02T16:23:09,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741836_1012 (size=7323) 2024-12-02T16:23:09,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741836_1012 (size=7323) 2024-12-02T16:23:09,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741834_1010 (size=2288) 2024-12-02T16:23:09,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741834_1010 (size=2288) 2024-12-02T16:23:09,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741834_1010 (size=2288) 2024-12-02T16:23:09,221 DEBUG [RS:0;e5d35c688fae:36399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T16:23:09,221 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T16:23:09,221 INFO [RS:0;e5d35c688fae:36399 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T16:23:09,221 INFO [RS:0;e5d35c688fae:36399 {}] hbase.ChoreService(370): Chore service for: regionserver/e5d35c688fae:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T16:23:09,222 INFO [RS:0;e5d35c688fae:36399 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T16:23:09,222 INFO [regionserver/e5d35c688fae:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T16:23:09,222 INFO [RS:0;e5d35c688fae:36399 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36399 2024-12-02T16:23:09,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T16:23:09,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36399-0x1009bb082170001, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e5d35c688fae,36399,1733156571440 2024-12-02T16:23:09,275 INFO [RS:0;e5d35c688fae:36399 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T16:23:09,286 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e5d35c688fae,36399,1733156571440] 2024-12-02T16:23:09,294 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e5d35c688fae,36399,1733156571440 already deleted, retry=false 2024-12-02T16:23:09,294 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of e5d35c688fae,36399,1733156571440 on e5d35c688fae,38905,1733156570684 2024-12-02T16:23:09,301 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure e5d35c688fae,36399,1733156571440, splitWal=true, meta=true 2024-12-02T16:23:09,304 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for e5d35c688fae,36399,1733156571440 (carryingMeta=true) e5d35c688fae,36399,1733156571440/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@33ad8376[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-02T16:23:09,305 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure e5d35c688fae,36399,1733156571440, splitWal=true, meta=true 2024-12-02T16:23:09,306 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure e5d35c688fae,36399,1733156571440, splitWal=true, meta=true, isMeta: true 2024-12-02T16:23:09,308 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting 2024-12-02T16:23:09,309 INFO [PEWorker-2 {}] master.SplitWALManager(105): e5d35c688fae,36399,1733156571440 WAL count=1, meta=true 2024-12-02T16:23:09,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta}] 2024-12-02T16:23:09,317 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=e5d35c688fae,35297,1733156571540 2024-12-02T16:23:09,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta, worker=e5d35c688fae,35297,1733156571540}] 2024-12-02T16:23:09,336 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T16:23:09,337 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1] 2024-12-02T16:23:09,339 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server e5d35c688fae:36399 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: e5d35c688fae/172.17.0.2:36399 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:09,339 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1, error=java.net.ConnectException: Call to address=e5d35c688fae:36399 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: e5d35c688fae/172.17.0.2:36399 2024-12-02T16:23:09,339 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1 is java.net.ConnectException: Connection refused 2024-12-02T16:23:09,339 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1 from cache 2024-12-02T16:23:09,340 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address e5d35c688fae:36399 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: e5d35c688fae/172.17.0.2:36399 2024-12-02T16:23:09,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36399-0x1009bb082170001, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T16:23:09,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36399-0x1009bb082170001, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T16:23:09,387 INFO [RS:0;e5d35c688fae:36399 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T16:23:09,387 INFO [RS:0;e5d35c688fae:36399 {}] regionserver.HRegionServer(1031): Exiting; stopping=e5d35c688fae,36399,1733156571440; zookeeper connection closed. 2024-12-02T16:23:09,387 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@52bed297 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@52bed297 2024-12-02T16:23:09,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35297 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-02T16:23:09,504 INFO [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta, size=7.2 K (7323bytes) 2024-12-02T16:23:09,504 INFO [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta 2024-12-02T16:23:09,505 INFO [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta after 1ms 2024-12-02T16:23:09,530 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:09,531 INFO [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta took 27ms 2024-12-02T16:23:09,538 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-02T16:23:09,538 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta so closing down 2024-12-02T16:23:09,538 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T16:23:09,540 INFO [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta.temp 2024-12-02T16:23:09,565 INFO [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta.temp 2024-12-02T16:23:09,566 INFO [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T16:23:09,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741892_1070 (size=7071) 2024-12-02T16:23:09,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741892_1070 (size=7071) 2024-12-02T16:23:09,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741892_1070 (size=7071) 2024-12-02T16:23:09,574 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-02T16:23:09,576 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta.temp to hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-02T16:23:09,576 INFO [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 45 ms; skipped=1; WAL=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false 2024-12-02T16:23:09,576 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta, journal: Splitting hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta, size=7.2 K (7323bytes) at 1733156589504Finishing writing output for hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta so closing down at 1733156589538 (+34 ms)Creating recovered edits writer path=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta.temp at 1733156589565 (+27 ms)3 split writer threads finished at 1733156589566 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733156589574 (+8 ms)Rename recovered edits hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta.temp to hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1733156589576 (+2 ms)Processed 16 edits across 1 Regions in 45 ms; skipped=1; WAL=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false at 1733156589576 2024-12-02T16:23:09,576 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta 2024-12-02T16:23:09,578 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-02T16:23:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-02T16:23:09,585 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta to hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/oldWALs 2024-12-02T16:23:09,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-02T16:23:09,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta, worker=e5d35c688fae,35297,1733156571540 in 265 msec 2024-12-02T16:23:09,590 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=e5d35c688fae,35297,1733156571540 2024-12-02T16:23:09,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-02T16:23:09,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure e5d35c688fae%2C36399%2C1733156571440.meta.1733156573508.meta, worker=e5d35c688fae,35297,1733156571540 in 280 msec 2024-12-02T16:23:09,596 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting dir is empty, no logs to split. 2024-12-02T16:23:09,596 INFO [PEWorker-2 {}] master.SplitWALManager(105): e5d35c688fae,36399,1733156571440 WAL count=0, meta=true 2024-12-02T16:23:09,596 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if e5d35c688fae,36399,1733156571440 WAL splitting is done? wals=0, meta=true 2024-12-02T16:23:09,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T16:23:09,600 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T16:23:09,601 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T16:23:09,646 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T16:23:09,647 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1] 2024-12-02T16:23:09,648 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to e5d35c688fae:36399 this server is in the failed servers list 2024-12-02T16:23:09,648 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=e5d35c688fae:36399 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e5d35c688fae:36399 2024-12-02T16:23:09,648 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e5d35c688fae:36399 2024-12-02T16:23:09,648 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1 from cache 2024-12-02T16:23:09,752 DEBUG [e5d35c688fae:38905 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-02T16:23:09,752 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(204): Hosts are {e5d35c688fae=0} racks are {/default-rack=0} 2024-12-02T16:23:09,752 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T16:23:09,752 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T16:23:09,752 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T16:23:09,752 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T16:23:09,752 INFO [e5d35c688fae:38905 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T16:23:09,752 INFO [e5d35c688fae:38905 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T16:23:09,752 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T16:23:09,753 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e5d35c688fae,35297,1733156571540 2024-12-02T16:23:09,754 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e5d35c688fae,35297,1733156571540, state=OPENING 2024-12-02T16:23:09,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T16:23:09,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T16:23:09,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T16:23:09,792 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T16:23:09,792 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T16:23:09,792 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T16:23:09,792 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T16:23:09,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e5d35c688fae,35297,1733156571540}] 2024-12-02T16:23:09,952 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T16:23:09,953 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:23:09,953 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-02T16:23:09,956 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e5d35c688fae%2C35297%2C1733156571540.meta, suffix=.meta, logDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,35297,1733156571540, archiveDir=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/oldWALs, maxLogs=32 2024-12-02T16:23:09,976 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,35297,1733156571540/e5d35c688fae%2C35297%2C1733156571540.meta.1733156589957.meta, exclude list is [], retry=0 2024-12-02T16:23:09,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:09,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:09,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:10,014 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,35297,1733156571540/e5d35c688fae%2C35297%2C1733156571540.meta.1733156589957.meta 2024-12-02T16:23:10,015 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:43815:43815)] 2024-12-02T16:23:10,015 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:10,015 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T16:23:10,015 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T16:23:10,016 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T16:23:10,016 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T16:23:10,016 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:10,016 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T16:23:10,016 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T16:23:10,017 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T16:23:10,019 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T16:23:10,019 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:10,019 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T16:23:10,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T16:23:10,020 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T16:23:10,021 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:10,021 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T16:23:10,021 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T16:23:10,022 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T16:23:10,022 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:10,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T16:23:10,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T16:23:10,024 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T16:23:10,024 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:10,024 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T16:23:10,024 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T16:23:10,025 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740 2024-12-02T16:23:10,027 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740 2024-12-02T16:23:10,027 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-02T16:23:10,055 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:10,059 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-02T16:23:10,059 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-12-02T16:23:10,080 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/.tmp/info/da4aff6bad214931abc4c9ff281534c4 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981./info:regioninfo/1733156588949/Put/seqid=0 2024-12-02T16:23:10,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741894_1072 (size=11177) 2024-12-02T16:23:10,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741894_1072 (size=11177) 2024-12-02T16:23:10,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741894_1072 (size=11177) 2024-12-02T16:23:10,087 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/.tmp/info/da4aff6bad214931abc4c9ff281534c4 2024-12-02T16:23:10,112 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/.tmp/ns/e7afc49d80394474b8b694049136918d is 43, key is default/ns:d/1733156573775/Put/seqid=0 2024-12-02T16:23:10,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741895_1073 (size=5153) 2024-12-02T16:23:10,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741895_1073 (size=5153) 2024-12-02T16:23:10,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741895_1073 (size=5153) 2024-12-02T16:23:10,120 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/.tmp/ns/e7afc49d80394474b8b694049136918d 2024-12-02T16:23:10,141 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/.tmp/table/68bb5b1983244edf8523ea2c3b4d852b is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733156587198/Put/seqid=0 2024-12-02T16:23:10,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741896_1074 (size=5431) 2024-12-02T16:23:10,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741896_1074 (size=5431) 2024-12-02T16:23:10,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741896_1074 (size=5431) 2024-12-02T16:23:10,148 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/.tmp/table/68bb5b1983244edf8523ea2c3b4d852b 2024-12-02T16:23:10,155 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/.tmp/info/da4aff6bad214931abc4c9ff281534c4 as hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/info/da4aff6bad214931abc4c9ff281534c4 2024-12-02T16:23:10,156 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T16:23:10,157 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,35297,1733156571540, seqNum=-1] 2024-12-02T16:23:10,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35297 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Get size: 119 connection: 172.17.0.2:44614 deadline: 1733156650158, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on e5d35c688fae,35297,1733156571540 2024-12-02T16:23:10,159 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,35297,1733156571540, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,35297,1733156571540, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on e5d35c688fae,35297,1733156571540 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T16:23:10,160 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,35297,1733156571540, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on e5d35c688fae,35297,1733156571540 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T16:23:10,160 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,35297,1733156571540, seqNum=-1 from cache 2024-12-02T16:23:10,164 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/info/da4aff6bad214931abc4c9ff281534c4, entries=36, sequenceid=18, filesize=10.9 K 2024-12-02T16:23:10,165 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/.tmp/ns/e7afc49d80394474b8b694049136918d as hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/ns/e7afc49d80394474b8b694049136918d 2024-12-02T16:23:10,172 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/ns/e7afc49d80394474b8b694049136918d, entries=2, sequenceid=18, filesize=5.0 K 2024-12-02T16:23:10,173 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/.tmp/table/68bb5b1983244edf8523ea2c3b4d852b as hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/table/68bb5b1983244edf8523ea2c3b4d852b 2024-12-02T16:23:10,180 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/table/68bb5b1983244edf8523ea2c3b4d852b, entries=2, sequenceid=18, filesize=5.3 K 2024-12-02T16:23:10,181 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 122ms, sequenceid=18, compaction requested=false; wal=null 2024-12-02T16:23:10,182 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-02T16:23:10,183 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T16:23:10,183 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T16:23:10,184 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T16:23:10,186 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T16:23:10,189 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-02T16:23:10,190 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67758244, jitterRate=0.009676516056060791}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T16:23:10,190 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T16:23:10,191 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733156590016Writing region info on filesystem at 1733156590016Initializing all the Stores at 1733156590017 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733156590017Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733156590017Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156590017Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733156590017Obtaining lock to block concurrent updates at 1733156590059 (+42 ms)Preparing flush snapshotting stores in 1588230740 at 1733156590059Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1733156590060 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733156590060Flushing 1588230740/info: creating writer at 1733156590060Flushing 1588230740/info: appending metadata at 1733156590080 (+20 ms)Flushing 1588230740/info: closing flushed file at 1733156590080Flushing 1588230740/ns: creating writer at 1733156590094 (+14 ms)Flushing 1588230740/ns: appending metadata at 1733156590111 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1733156590112 (+1 ms)Flushing 1588230740/table: creating writer at 1733156590126 (+14 ms)Flushing 1588230740/table: appending metadata at 1733156590141 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733156590141Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5379a8a6: reopening flushed file at 1733156590154 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1102e66c: reopening flushed file at 1733156590164 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8b1ce12: reopening flushed file at 1733156590172 (+8 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 122ms, sequenceid=18, compaction requested=false; wal=null at 1733156590181 (+9 ms)Cleaning up temporary data from old regions at 1733156590183 (+2 ms)Running coprocessor post-open hooks at 1733156590190 (+7 ms)Region opened successfully at 1733156590191 (+1 ms) 2024-12-02T16:23:10,192 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1733156589946 2024-12-02T16:23:10,195 DEBUG [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T16:23:10,195 INFO [RS_OPEN_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T16:23:10,196 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=e5d35c688fae,35297,1733156571540 2024-12-02T16:23:10,196 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e5d35c688fae,35297,1733156571540, state=OPEN 2024-12-02T16:23:10,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T16:23:10,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T16:23:10,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T16:23:10,202 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e5d35c688fae,35297,1733156571540 2024-12-02T16:23:10,202 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T16:23:10,202 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T16:23:10,202 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T16:23:10,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-02T16:23:10,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e5d35c688fae,35297,1733156571540 in 410 msec 2024-12-02T16:23:10,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-02T16:23:10,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 608 msec 2024-12-02T16:23:10,210 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): e5d35c688fae,36399,1733156571440 had 2 regions 2024-12-02T16:23:10,211 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure e5d35c688fae,36399,1733156571440, splitWal=true, meta=true, isMeta: false 2024-12-02T16:23:10,213 INFO [PEWorker-1 {}] master.SplitWALManager(105): e5d35c688fae,36399,1733156571440 WAL count=1, meta=false 2024-12-02T16:23:10,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure e5d35c688fae%2C36399%2C1733156571440.1733156573042}] 2024-12-02T16:23:10,215 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=e5d35c688fae,43901,1733156571586 2024-12-02T16:23:10,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure e5d35c688fae%2C36399%2C1733156571440.1733156573042, worker=e5d35c688fae,43901,1733156571586}] 2024-12-02T16:23:10,369 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T16:23:10,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52431, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T16:23:10,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43901 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-12-02T16:23:10,390 INFO [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.1733156573042, size=2.2 K (2288bytes) 2024-12-02T16:23:10,390 INFO [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.1733156573042 2024-12-02T16:23:10,391 INFO [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.1733156573042 after 0ms 2024-12-02T16:23:10,432 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:10,432 INFO [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.1733156573042 took 42ms 2024-12-02T16:23:10,436 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 01e69ba503cdc68fb29228453655d981: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-02T16:23:10,436 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.1733156573042 so closing down 2024-12-02T16:23:10,436 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T16:23:10,436 INFO [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T16:23:10,436 INFO [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.1733156573042, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-12-02T16:23:10,436 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.1733156573042, journal: Splitting hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.1733156573042, size=2.2 K (2288bytes) at 1733156590390Finishing writing output for hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.1733156573042 so closing down at 1733156590436 (+46 ms)3 split writer threads finished at 1733156590436Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.1733156573042, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1733156590436 2024-12-02T16:23:10,436 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.1733156573042 2024-12-02T16:23:10,437 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e5d35c688fae:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-02T16:23:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38905 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-12-02T16:23:10,442 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting/e5d35c688fae%2C36399%2C1733156571440.1733156573042 to hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/oldWALs 2024-12-02T16:23:10,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-02T16:23:10,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e5d35c688fae%2C36399%2C1733156571440.1733156573042, worker=e5d35c688fae,43901,1733156571586 in 226 msec 2024-12-02T16:23:10,447 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=e5d35c688fae,43901,1733156571586 2024-12-02T16:23:10,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-12-02T16:23:10,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure e5d35c688fae%2C36399%2C1733156571440.1733156573042, worker=e5d35c688fae,43901,1733156571586 in 234 msec 2024-12-02T16:23:10,452 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,36399,1733156571440-splitting dir is empty, no logs to split. 2024-12-02T16:23:10,452 INFO [PEWorker-1 {}] master.SplitWALManager(105): e5d35c688fae,36399,1733156571440 WAL count=0, meta=false 2024-12-02T16:23:10,452 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if e5d35c688fae,36399,1733156571440 WAL splitting is done? wals=0, meta=false 2024-12-02T16:23:10,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, ASSIGN}] 2024-12-02T16:23:10,456 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, ASSIGN 2024-12-02T16:23:10,457 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T16:23:10,608 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(204): Hosts are {e5d35c688fae=0} racks are {/default-rack=0} 2024-12-02T16:23:10,608 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T16:23:10,608 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T16:23:10,608 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T16:23:10,608 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T16:23:10,608 INFO [e5d35c688fae:38905 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T16:23:10,608 INFO [e5d35c688fae:38905 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T16:23:10,608 DEBUG [e5d35c688fae:38905 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T16:23:10,609 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=01e69ba503cdc68fb29228453655d981, regionState=OPENING, regionLocation=e5d35c688fae,35297,1733156571540 2024-12-02T16:23:10,610 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server e5d35c688fae:36399 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e5d35c688fae/172.17.0.2:36399 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:10,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1, error=java.net.ConnectException: Call to address=e5d35c688fae:36399 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e5d35c688fae/172.17.0.2:36399 2024-12-02T16:23:10,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-02T16:23:10,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,36399,1733156571440, seqNum=-1 from cache 2024-12-02T16:23:10,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address e5d35c688fae:36399 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e5d35c688fae/172.17.0.2:36399 2024-12-02T16:23:10,716 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T16:23:10,717 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,35297,1733156571540, seqNum=-1] 2024-12-02T16:23:10,717 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T16:23:10,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56675, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T16:23:10,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, ASSIGN because future has completed 2024-12-02T16:23:10,723 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,35297,1733156571540}] 2024-12-02T16:23:10,885 INFO [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:10,886 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 01e69ba503cdc68fb29228453655d981, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:10,886 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:10,886 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:10,887 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:10,887 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:10,889 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:10,891 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01e69ba503cdc68fb29228453655d981 columnFamilyName cf1 2024-12-02T16:23:10,891 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:10,902 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf1/9351bcb6240441149880e3136a6424ca 2024-12-02T16:23:10,902 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(327): Store=01e69ba503cdc68fb29228453655d981/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:10,903 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:10,904 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01e69ba503cdc68fb29228453655d981 columnFamilyName cf2 2024-12-02T16:23:10,904 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:10,911 DEBUG [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/cf2/188e2968f4a14a279d18ec5e26d7bff9 2024-12-02T16:23:10,912 INFO [StoreOpener-01e69ba503cdc68fb29228453655d981-1 {}] regionserver.HStore(327): Store=01e69ba503cdc68fb29228453655d981/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:10,912 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:10,913 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:10,914 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:10,915 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:10,915 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:10,915 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-02T16:23:10,917 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:10,917 INFO [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 01e69ba503cdc68fb29228453655d981; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74151457, jitterRate=0.10494281351566315}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-02T16:23:10,918 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:10,918 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 01e69ba503cdc68fb29228453655d981: Running coprocessor pre-open hook at 1733156590887Writing region info on filesystem at 1733156590887Initializing all the Stores at 1733156590889 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156590889Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156590889Cleaning up temporary data from old regions at 1733156590915 (+26 ms)Running coprocessor post-open hooks at 1733156590918 (+3 ms)Region opened successfully at 1733156590918 2024-12-02T16:23:10,919 INFO [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., pid=21, masterSystemTime=1733156590876 2024-12-02T16:23:10,921 DEBUG [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:10,922 INFO [RS_OPEN_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:10,922 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=01e69ba503cdc68fb29228453655d981, regionState=OPEN, openSeqNum=18, regionLocation=e5d35c688fae,35297,1733156571540 2024-12-02T16:23:10,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,35297,1733156571540 because future has completed 2024-12-02T16:23:10,928 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-02T16:23:10,928 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 01e69ba503cdc68fb29228453655d981, server=e5d35c688fae,35297,1733156571540 in 203 msec 2024-12-02T16:23:10,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-12-02T16:23:10,931 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server e5d35c688fae,36399,1733156571440 after splitting done 2024-12-02T16:23:10,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=01e69ba503cdc68fb29228453655d981, ASSIGN in 473 msec 2024-12-02T16:23:10,934 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure e5d35c688fae,36399,1733156571440, splitWal=true, meta=true in 1.6360 sec 2024-12-02T16:23:11,010 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-02T16:23:11,010 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-02T16:23:11,011 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-02T16:23:11,011 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-02T16:23:11,013 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-02T16:23:11,013 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-02T16:23:11,166 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T16:23:11,168 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5d35c688fae,35297,1733156571540, seqNum=-1] 2024-12-02T16:23:11,286 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to e5d35c688fae:36399 this server is in the failed servers list 2024-12-02T16:23:11,286 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., hostname=e5d35c688fae,36399,1733156571440, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., hostname=e5d35c688fae,36399,1733156571440, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=e5d35c688fae:36399 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e5d35c688fae:36399 2024-12-02T16:23:11,286 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., hostname=e5d35c688fae,36399,1733156571440, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e5d35c688fae:36399 2024-12-02T16:23:11,286 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., hostname=e5d35c688fae,36399,1733156571440, seqNum=12 from cache 2024-12-02T16:23:11,505 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., hostname=e5d35c688fae,35297,1733156571540, seqNum=18] 2024-12-02T16:23:11,520 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=403 (was 410), OpenFileDescriptor=1046 (was 1022) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=246 (was 241) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5948 (was 6077) 2024-12-02T16:23:11,521 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1046 is superior to 1024 2024-12-02T16:23:11,533 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=403, OpenFileDescriptor=1046, MaxFileDescriptor=1048576, SystemLoadAverage=246, ProcessCount=11, AvailableMemoryMB=5947 2024-12-02T16:23:11,533 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1046 is superior to 1024 2024-12-02T16:23:11,547 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:23:11,549 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:23:11,550 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T16:23:11,553 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-93547014, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/hregion-93547014, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:11,565 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-93547014/hregion-93547014.1733156591553, exclude list is [], retry=0 2024-12-02T16:23:11,568 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:11,568 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:11,568 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:11,592 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-93547014/hregion-93547014.1733156591553 2024-12-02T16:23:11,593 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:33821:33821)] 2024-12-02T16:23:11,593 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 14dcc3388772665a7517bc804c8d5d38, NAME => 'testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33443/hbase 2024-12-02T16:23:11,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741898_1076 (size=67) 2024-12-02T16:23:11,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741898_1076 (size=67) 2024-12-02T16:23:11,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741898_1076 (size=67) 2024-12-02T16:23:11,613 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:11,614 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,616 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14dcc3388772665a7517bc804c8d5d38 columnFamilyName a 2024-12-02T16:23:11,616 DEBUG [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:11,617 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(327): Store=14dcc3388772665a7517bc804c8d5d38/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:11,617 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,618 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14dcc3388772665a7517bc804c8d5d38 columnFamilyName b 2024-12-02T16:23:11,619 DEBUG [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:11,619 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(327): Store=14dcc3388772665a7517bc804c8d5d38/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:11,619 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,621 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14dcc3388772665a7517bc804c8d5d38 columnFamilyName c 2024-12-02T16:23:11,621 DEBUG [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:11,622 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(327): Store=14dcc3388772665a7517bc804c8d5d38/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:11,622 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,623 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,623 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,625 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,625 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,625 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:11,627 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,630 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T16:23:11,631 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 14dcc3388772665a7517bc804c8d5d38; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74544580, jitterRate=0.11080080270767212}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:11,632 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 14dcc3388772665a7517bc804c8d5d38: Writing region info on filesystem at 1733156591613Initializing all the Stores at 1733156591614 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156591614Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156591614Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156591614Cleaning up temporary data from old regions at 1733156591625 (+11 ms)Region opened successfully at 1733156591632 (+7 ms) 2024-12-02T16:23:11,632 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 14dcc3388772665a7517bc804c8d5d38, disabling compactions & flushes 2024-12-02T16:23:11,632 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38. 2024-12-02T16:23:11,633 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38. 2024-12-02T16:23:11,633 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38. after waiting 0 ms 2024-12-02T16:23:11,633 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38. 2024-12-02T16:23:11,633 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38. 2024-12-02T16:23:11,633 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 14dcc3388772665a7517bc804c8d5d38: Waiting for close lock at 1733156591632Disabling compacts and flushes for region at 1733156591632Disabling writes for close at 1733156591633 (+1 ms)Writing region close event to WAL at 1733156591633Closed at 1733156591633 2024-12-02T16:23:11,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741897_1075 (size=175) 2024-12-02T16:23:11,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741897_1075 (size=175) 2024-12-02T16:23:11,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741897_1075 (size=175) 2024-12-02T16:23:11,640 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T16:23:11,640 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-93547014:(num 1733156591553) 2024-12-02T16:23:11,640 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:23:11,642 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:11,654 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591642, exclude list is [], retry=0 2024-12-02T16:23:11,657 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:11,657 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:11,658 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:11,682 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591642 2024-12-02T16:23:11,684 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:43815:43815)] 2024-12-02T16:23:11,684 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 14dcc3388772665a7517bc804c8d5d38, NAME => 'testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:11,684 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:11,684 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,684 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,686 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,687 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14dcc3388772665a7517bc804c8d5d38 columnFamilyName a 2024-12-02T16:23:11,687 DEBUG [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:11,688 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(327): Store=14dcc3388772665a7517bc804c8d5d38/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:11,688 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,689 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14dcc3388772665a7517bc804c8d5d38 columnFamilyName b 2024-12-02T16:23:11,689 DEBUG [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:11,689 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(327): Store=14dcc3388772665a7517bc804c8d5d38/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:11,689 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,690 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14dcc3388772665a7517bc804c8d5d38 columnFamilyName c 2024-12-02T16:23:11,690 DEBUG [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:11,691 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(327): Store=14dcc3388772665a7517bc804c8d5d38/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:11,691 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,692 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,693 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,694 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,694 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,695 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:11,696 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:11,697 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 14dcc3388772665a7517bc804c8d5d38; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69787497, jitterRate=0.03991474211215973}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:11,697 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 14dcc3388772665a7517bc804c8d5d38: Writing region info on filesystem at 1733156591684Initializing all the Stores at 1733156591685 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156591685Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156591686 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156591686Cleaning up temporary data from old regions at 1733156591694 (+8 ms)Region opened successfully at 1733156591697 (+3 ms) 2024-12-02T16:23:11,742 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 14dcc3388772665a7517bc804c8d5d38 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-02T16:23:11,759 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/.tmp/a/48d0f77308a4443c87156a83bfd01b16 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733156591697/Put/seqid=0 2024-12-02T16:23:11,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741900_1078 (size=5958) 2024-12-02T16:23:11,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741900_1078 (size=5958) 2024-12-02T16:23:11,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741900_1078 (size=5958) 2024-12-02T16:23:11,766 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/.tmp/a/48d0f77308a4443c87156a83bfd01b16 2024-12-02T16:23:11,784 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/.tmp/b/18f64076e1aa4dd481d7638f06ca4099 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733156591706/Put/seqid=0 2024-12-02T16:23:11,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741901_1079 (size=5958) 2024-12-02T16:23:11,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741901_1079 (size=5958) 2024-12-02T16:23:11,792 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/.tmp/b/18f64076e1aa4dd481d7638f06ca4099 2024-12-02T16:23:11,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741901_1079 (size=5958) 2024-12-02T16:23:11,811 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/.tmp/c/1ece71c44a524208b6fbe7ab704e1588 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733156591733/Put/seqid=0 2024-12-02T16:23:11,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741902_1080 (size=5958) 2024-12-02T16:23:11,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741902_1080 (size=5958) 2024-12-02T16:23:11,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741902_1080 (size=5958) 2024-12-02T16:23:11,819 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/.tmp/c/1ece71c44a524208b6fbe7ab704e1588 2024-12-02T16:23:11,824 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/.tmp/a/48d0f77308a4443c87156a83bfd01b16 as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/a/48d0f77308a4443c87156a83bfd01b16 2024-12-02T16:23:11,829 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/a/48d0f77308a4443c87156a83bfd01b16, entries=10, sequenceid=33, filesize=5.8 K 2024-12-02T16:23:11,830 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/.tmp/b/18f64076e1aa4dd481d7638f06ca4099 as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/b/18f64076e1aa4dd481d7638f06ca4099 2024-12-02T16:23:11,835 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/b/18f64076e1aa4dd481d7638f06ca4099, entries=10, sequenceid=33, filesize=5.8 K 2024-12-02T16:23:11,836 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/.tmp/c/1ece71c44a524208b6fbe7ab704e1588 as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/c/1ece71c44a524208b6fbe7ab704e1588 2024-12-02T16:23:11,841 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/c/1ece71c44a524208b6fbe7ab704e1588, entries=10, sequenceid=33, filesize=5.8 K 2024-12-02T16:23:11,842 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 14dcc3388772665a7517bc804c8d5d38 in 100ms, sequenceid=33, compaction requested=false 2024-12-02T16:23:11,842 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 14dcc3388772665a7517bc804c8d5d38: 2024-12-02T16:23:11,842 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 14dcc3388772665a7517bc804c8d5d38, disabling compactions & flushes 2024-12-02T16:23:11,842 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38. 2024-12-02T16:23:11,842 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38. 2024-12-02T16:23:11,842 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38. after waiting 0 ms 2024-12-02T16:23:11,842 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38. 2024-12-02T16:23:11,844 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38. 2024-12-02T16:23:11,844 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 14dcc3388772665a7517bc804c8d5d38: Waiting for close lock at 1733156591842Disabling compacts and flushes for region at 1733156591842Disabling writes for close at 1733156591842Writing region close event to WAL at 1733156591844 (+2 ms)Closed at 1733156591844 2024-12-02T16:23:11,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741899_1077 (size=6311) 2024-12-02T16:23:11,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741899_1077 (size=6311) 2024-12-02T16:23:11,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741899_1077 (size=6311) 2024-12-02T16:23:11,849 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/b/18f64076e1aa4dd481d7638f06ca4099 to hdfs://localhost:33443/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/b/18f64076e1aa4dd481d7638f06ca4099 2024-12-02T16:23:11,920 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33443/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591642, size=6.2 K (6311bytes) 2024-12-02T16:23:11,920 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33443/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591642 2024-12-02T16:23:11,920 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33443/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591642 after 0ms 2024-12-02T16:23:11,943 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:11,943 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33443/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591642 took 24ms 2024-12-02T16:23:11,946 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33443/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591642 so closing down 2024-12-02T16:23:11,946 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T16:23:11,947 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733156591642.temp 2024-12-02T16:23:11,971 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/recovered.edits/0000000000000000003-wal.1733156591642.temp 2024-12-02T16:23:11,972 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T16:23:11,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741903_1081 (size=5575) 2024-12-02T16:23:11,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741903_1081 (size=5575) 2024-12-02T16:23:11,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741903_1081 (size=5575) 2024-12-02T16:23:11,980 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/recovered.edits/0000000000000000003-wal.1733156591642.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-02T16:23:11,981 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/recovered.edits/0000000000000000003-wal.1733156591642.temp to hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/recovered.edits/0000000000000000032 2024-12-02T16:23:11,981 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 37 ms; skipped=2; WAL=hdfs://localhost:33443/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591642, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-12-02T16:23:11,981 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33443/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591642, journal: Splitting hdfs://localhost:33443/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591642, size=6.2 K (6311bytes) at 1733156591920Finishing writing output for hdfs://localhost:33443/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591642 so closing down at 1733156591946 (+26 ms)Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/recovered.edits/0000000000000000003-wal.1733156591642.temp at 1733156591972 (+26 ms)3 split writer threads finished at 1733156591972Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/recovered.edits/0000000000000000003-wal.1733156591642.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733156591980 (+8 ms)Rename recovered edits hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/recovered.edits/0000000000000000003-wal.1733156591642.temp to hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/recovered.edits/0000000000000000032 at 1733156591981 (+1 ms)Processed 32 edits across 1 Regions in 37 ms; skipped=2; WAL=hdfs://localhost:33443/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591642, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1733156591981 2024-12-02T16:23:11,983 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33443/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591642 to hdfs://localhost:33443/hbase/oldWALs/wal.1733156591642 2024-12-02T16:23:11,984 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/recovered.edits/0000000000000000032 2024-12-02T16:23:11,984 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:23:11,986 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:11,997 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591986, exclude list is [], retry=0 2024-12-02T16:23:12,000 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:12,001 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:12,001 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:12,024 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591986 2024-12-02T16:23:12,024 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:43815:43815)] 2024-12-02T16:23:12,024 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 14dcc3388772665a7517bc804c8d5d38, NAME => 'testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:12,024 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:12,024 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:12,024 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:12,026 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:12,027 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14dcc3388772665a7517bc804c8d5d38 columnFamilyName a 2024-12-02T16:23:12,027 DEBUG [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:12,032 DEBUG [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/a/48d0f77308a4443c87156a83bfd01b16 2024-12-02T16:23:12,032 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(327): Store=14dcc3388772665a7517bc804c8d5d38/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:12,032 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:12,034 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14dcc3388772665a7517bc804c8d5d38 columnFamilyName b 2024-12-02T16:23:12,034 DEBUG [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:12,035 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(327): Store=14dcc3388772665a7517bc804c8d5d38/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:12,035 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:12,036 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14dcc3388772665a7517bc804c8d5d38 columnFamilyName c 2024-12-02T16:23:12,036 DEBUG [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:12,042 DEBUG [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/c/1ece71c44a524208b6fbe7ab704e1588 2024-12-02T16:23:12,043 INFO [StoreOpener-14dcc3388772665a7517bc804c8d5d38-1 {}] regionserver.HStore(327): Store=14dcc3388772665a7517bc804c8d5d38/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:12,043 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:12,044 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:12,045 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:12,046 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/recovered.edits/0000000000000000032 2024-12-02T16:23:12,068 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:12,069 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/recovered.edits/0000000000000000032 2024-12-02T16:23:12,069 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 14dcc3388772665a7517bc804c8d5d38 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-02T16:23:12,081 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/.tmp/b/8306cec50cc64a83acbb826023e0df54 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733156591706/Put/seqid=0 2024-12-02T16:23:12,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741905_1083 (size=5958) 2024-12-02T16:23:12,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741905_1083 (size=5958) 2024-12-02T16:23:12,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741905_1083 (size=5958) 2024-12-02T16:23:12,088 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/.tmp/b/8306cec50cc64a83acbb826023e0df54 2024-12-02T16:23:12,094 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/.tmp/b/8306cec50cc64a83acbb826023e0df54 as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/b/8306cec50cc64a83acbb826023e0df54 2024-12-02T16:23:12,099 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/b/8306cec50cc64a83acbb826023e0df54, entries=10, sequenceid=32, filesize=5.8 K 2024-12-02T16:23:12,099 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 14dcc3388772665a7517bc804c8d5d38 in 30ms, sequenceid=32, compaction requested=false; wal=null 2024-12-02T16:23:12,100 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/recovered.edits/0000000000000000032 2024-12-02T16:23:12,101 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:12,101 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:12,102 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:12,103 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 14dcc3388772665a7517bc804c8d5d38 2024-12-02T16:23:12,106 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/14dcc3388772665a7517bc804c8d5d38/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-02T16:23:12,107 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 14dcc3388772665a7517bc804c8d5d38; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65541291, jitterRate=-0.023358657956123352}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:12,108 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 14dcc3388772665a7517bc804c8d5d38: Writing region info on filesystem at 1733156592025Initializing all the Stores at 1733156592025Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156592025Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156592026 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156592026Obtaining lock to block concurrent updates at 1733156592069 (+43 ms)Preparing flush snapshotting stores in 14dcc3388772665a7517bc804c8d5d38 at 1733156592069Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733156592069Flushing stores of testReplayEditsWrittenViaHRegion,,1733156591548.14dcc3388772665a7517bc804c8d5d38. at 1733156592069Flushing 14dcc3388772665a7517bc804c8d5d38/b: creating writer at 1733156592069Flushing 14dcc3388772665a7517bc804c8d5d38/b: appending metadata at 1733156592080 (+11 ms)Flushing 14dcc3388772665a7517bc804c8d5d38/b: closing flushed file at 1733156592080Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b9159c8: reopening flushed file at 1733156592093 (+13 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 14dcc3388772665a7517bc804c8d5d38 in 30ms, sequenceid=32, compaction requested=false; wal=null at 1733156592099 (+6 ms)Cleaning up temporary data from old regions at 1733156592101 (+2 ms)Region opened successfully at 1733156592108 (+7 ms) 2024-12-02T16:23:12,132 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=413 (was 403) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1520450069-172.17.0.2-1733156567009:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:36460 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1520450069-172.17.0.2-1733156567009:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1520450069-172.17.0.2-1733156567009:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:37244 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:37022 [Waiting for operation #32] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:45644 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:36284 [Waiting for operation #54] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:45456 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1116 (was 1046) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=282 (was 246) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5846 (was 5947) 2024-12-02T16:23:12,133 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1116 is superior to 1024 2024-12-02T16:23:12,146 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=413, OpenFileDescriptor=1116, MaxFileDescriptor=1048576, SystemLoadAverage=282, ProcessCount=11, AvailableMemoryMB=5846 2024-12-02T16:23:12,146 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1116 is superior to 1024 2024-12-02T16:23:12,163 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:23:12,165 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:23:12,166 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T16:23:12,168 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-69149219, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/hregion-69149219, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:12,180 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-69149219/hregion-69149219.1733156592168, exclude list is [], retry=0 2024-12-02T16:23:12,185 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:12,185 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:12,188 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:12,213 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-69149219/hregion-69149219.1733156592168 2024-12-02T16:23:12,214 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:33821:33821)] 2024-12-02T16:23:12,214 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => bac3f6700c756361834801463992d240, NAME => 'testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33443/hbase 2024-12-02T16:23:12,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741907_1085 (size=68) 2024-12-02T16:23:12,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741907_1085 (size=68) 2024-12-02T16:23:12,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741907_1085 (size=68) 2024-12-02T16:23:12,223 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:12,224 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bac3f6700c756361834801463992d240 2024-12-02T16:23:12,226 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bac3f6700c756361834801463992d240 columnFamilyName a 2024-12-02T16:23:12,226 DEBUG [StoreOpener-bac3f6700c756361834801463992d240-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:12,227 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(327): Store=bac3f6700c756361834801463992d240/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:12,227 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bac3f6700c756361834801463992d240 2024-12-02T16:23:12,228 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bac3f6700c756361834801463992d240 columnFamilyName b 2024-12-02T16:23:12,228 DEBUG [StoreOpener-bac3f6700c756361834801463992d240-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:12,229 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(327): Store=bac3f6700c756361834801463992d240/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:12,229 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bac3f6700c756361834801463992d240 2024-12-02T16:23:12,230 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bac3f6700c756361834801463992d240 columnFamilyName c 2024-12-02T16:23:12,231 DEBUG [StoreOpener-bac3f6700c756361834801463992d240-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:12,231 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(327): Store=bac3f6700c756361834801463992d240/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:12,231 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bac3f6700c756361834801463992d240 2024-12-02T16:23:12,232 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240 2024-12-02T16:23:12,232 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240 2024-12-02T16:23:12,233 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bac3f6700c756361834801463992d240 2024-12-02T16:23:12,234 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bac3f6700c756361834801463992d240 2024-12-02T16:23:12,234 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:12,235 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bac3f6700c756361834801463992d240 2024-12-02T16:23:12,237 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T16:23:12,238 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bac3f6700c756361834801463992d240; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66516942, jitterRate=-0.008820325136184692}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:12,238 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bac3f6700c756361834801463992d240: Writing region info on filesystem at 1733156592223Initializing all the Stores at 1733156592224 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156592224Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156592224Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156592224Cleaning up temporary data from old regions at 1733156592234 (+10 ms)Region opened successfully at 1733156592238 (+4 ms) 2024-12-02T16:23:12,238 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bac3f6700c756361834801463992d240, disabling compactions & flushes 2024-12-02T16:23:12,238 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240. 2024-12-02T16:23:12,239 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240. 2024-12-02T16:23:12,239 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240. after waiting 0 ms 2024-12-02T16:23:12,239 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240. 2024-12-02T16:23:12,239 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240. 2024-12-02T16:23:12,239 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bac3f6700c756361834801463992d240: Waiting for close lock at 1733156592238Disabling compacts and flushes for region at 1733156592238Disabling writes for close at 1733156592239 (+1 ms)Writing region close event to WAL at 1733156592239Closed at 1733156592239 2024-12-02T16:23:12,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741906_1084 (size=175) 2024-12-02T16:23:12,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741906_1084 (size=175) 2024-12-02T16:23:12,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741906_1084 (size=175) 2024-12-02T16:23:12,244 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T16:23:12,244 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-69149219:(num 1733156592168) 2024-12-02T16:23:12,244 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:23:12,245 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:12,256 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156592246, exclude list is [], retry=0 2024-12-02T16:23:12,259 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:12,259 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:12,259 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:12,283 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156592246 2024-12-02T16:23:12,283 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:43815:43815)] 2024-12-02T16:23:12,333 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bac3f6700c756361834801463992d240, NAME => 'testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:12,335 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush bac3f6700c756361834801463992d240 2024-12-02T16:23:12,335 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:12,335 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bac3f6700c756361834801463992d240 2024-12-02T16:23:12,335 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bac3f6700c756361834801463992d240 2024-12-02T16:23:12,337 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bac3f6700c756361834801463992d240 2024-12-02T16:23:12,338 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bac3f6700c756361834801463992d240 columnFamilyName a 2024-12-02T16:23:12,338 DEBUG [StoreOpener-bac3f6700c756361834801463992d240-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:12,338 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(327): Store=bac3f6700c756361834801463992d240/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:12,339 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bac3f6700c756361834801463992d240 2024-12-02T16:23:12,339 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bac3f6700c756361834801463992d240 columnFamilyName b 2024-12-02T16:23:12,339 DEBUG [StoreOpener-bac3f6700c756361834801463992d240-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:12,340 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(327): Store=bac3f6700c756361834801463992d240/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:12,340 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bac3f6700c756361834801463992d240 2024-12-02T16:23:12,341 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bac3f6700c756361834801463992d240 columnFamilyName c 2024-12-02T16:23:12,341 DEBUG [StoreOpener-bac3f6700c756361834801463992d240-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:12,341 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(327): Store=bac3f6700c756361834801463992d240/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:12,341 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bac3f6700c756361834801463992d240 2024-12-02T16:23:12,342 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240 2024-12-02T16:23:12,343 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240 2024-12-02T16:23:12,344 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bac3f6700c756361834801463992d240 2024-12-02T16:23:12,344 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bac3f6700c756361834801463992d240 2024-12-02T16:23:12,345 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:12,346 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bac3f6700c756361834801463992d240 2024-12-02T16:23:12,346 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bac3f6700c756361834801463992d240; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73928601, jitterRate=0.10162200033664703}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:12,346 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bac3f6700c756361834801463992d240 2024-12-02T16:23:12,347 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bac3f6700c756361834801463992d240: Running coprocessor pre-open hook at 1733156592335Writing region info on filesystem at 1733156592335Initializing all the Stores at 1733156592336 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156592336Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156592336Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156592336Cleaning up temporary data from old regions at 1733156592344 (+8 ms)Running coprocessor post-open hooks at 1733156592346 (+2 ms)Region opened successfully at 1733156592347 (+1 ms) 2024-12-02T16:23:12,360 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bac3f6700c756361834801463992d240 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-02T16:23:12,361 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bac3f6700c756361834801463992d240/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:12,685 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T16:23:13,362 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bac3f6700c756361834801463992d240/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:14,362 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bac3f6700c756361834801463992d240/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:15,363 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bac3f6700c756361834801463992d240/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:16,365 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bac3f6700c756361834801463992d240/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:17,365 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bac3f6700c756361834801463992d240/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:17,840 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T16:23:18,366 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bac3f6700c756361834801463992d240/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:19,367 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bac3f6700c756361834801463992d240/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:19,882 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T16:23:20,368 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bac3f6700c756361834801463992d240/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:21,010 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-02T16:23:21,010 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-02T16:23:21,011 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-02T16:23:21,012 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-02T16:23:21,370 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bac3f6700c756361834801463992d240/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:21,373 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bac3f6700c756361834801463992d240: 2024-12-02T16:23:21,373 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240. 2024-12-02T16:23:21,389 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bac3f6700c756361834801463992d240: 2024-12-02T16:23:21,390 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-02T16:23:21,390 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bac3f6700c756361834801463992d240, disabling compactions & flushes 2024-12-02T16:23:21,390 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240. 2024-12-02T16:23:21,390 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240. 2024-12-02T16:23:21,390 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240. after waiting 0 ms 2024-12-02T16:23:21,390 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240. 2024-12-02T16:23:21,390 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240. 2024-12-02T16:23:21,390 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240. 2024-12-02T16:23:21,390 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bac3f6700c756361834801463992d240: Waiting for close lock at 1733156601390Running coprocessor pre-close hooks at 1733156601390Disabling compacts and flushes for region at 1733156601390Disabling writes for close at 1733156601390Writing region close event to WAL at 1733156601390Running coprocessor post-close hooks at 1733156601390Closed at 1733156601390 2024-12-02T16:23:21,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741908_1086 (size=4236) 2024-12-02T16:23:21,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741908_1086 (size=4236) 2024-12-02T16:23:21,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741908_1086 (size=4236) 2024-12-02T16:23:21,407 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33443/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156592246, size=4.1 K (4236bytes) 2024-12-02T16:23:21,407 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33443/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156592246 2024-12-02T16:23:21,407 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33443/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156592246 after 0ms 2024-12-02T16:23:21,431 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:21,431 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33443/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156592246 took 24ms 2024-12-02T16:23:21,433 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33443/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156592246 so closing down 2024-12-02T16:23:21,434 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T16:23:21,435 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733156592246.temp 2024-12-02T16:23:21,458 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/recovered.edits/0000000000000000004-wal.1733156592246.temp 2024-12-02T16:23:21,458 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T16:23:21,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741909_1087 (size=3245) 2024-12-02T16:23:21,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741909_1087 (size=3245) 2024-12-02T16:23:21,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741909_1087 (size=3245) 2024-12-02T16:23:21,465 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/recovered.edits/0000000000000000004-wal.1733156592246.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-02T16:23:21,466 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/recovered.edits/0000000000000000004-wal.1733156592246.temp to hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/recovered.edits/0000000000000000026 2024-12-02T16:23:21,467 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 35 ms; skipped=3; WAL=hdfs://localhost:33443/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156592246, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-12-02T16:23:21,467 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33443/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156592246, journal: Splitting hdfs://localhost:33443/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156592246, size=4.1 K (4236bytes) at 1733156601407Finishing writing output for hdfs://localhost:33443/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156592246 so closing down at 1733156601433 (+26 ms)Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/recovered.edits/0000000000000000004-wal.1733156592246.temp at 1733156601458 (+25 ms)3 split writer threads finished at 1733156601459 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/recovered.edits/0000000000000000004-wal.1733156592246.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733156601465 (+6 ms)Rename recovered edits hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/recovered.edits/0000000000000000004-wal.1733156592246.temp to hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/recovered.edits/0000000000000000026 at 1733156601466 (+1 ms)Processed 23 edits across 1 Regions in 35 ms; skipped=3; WAL=hdfs://localhost:33443/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156592246, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1733156601467 (+1 ms) 2024-12-02T16:23:21,468 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33443/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156592246 to hdfs://localhost:33443/hbase/oldWALs/wal.1733156592246 2024-12-02T16:23:21,469 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/recovered.edits/0000000000000000026 2024-12-02T16:23:21,469 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:23:21,471 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:21,482 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156601471, exclude list is [], retry=0 2024-12-02T16:23:21,485 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:21,485 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:21,486 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:21,509 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156601471 2024-12-02T16:23:21,509 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:35029:35029)] 2024-12-02T16:23:21,509 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bac3f6700c756361834801463992d240, NAME => 'testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:21,510 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush bac3f6700c756361834801463992d240 2024-12-02T16:23:21,510 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:21,510 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bac3f6700c756361834801463992d240 2024-12-02T16:23:21,510 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bac3f6700c756361834801463992d240 2024-12-02T16:23:21,512 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bac3f6700c756361834801463992d240 2024-12-02T16:23:21,513 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bac3f6700c756361834801463992d240 columnFamilyName a 2024-12-02T16:23:21,513 DEBUG [StoreOpener-bac3f6700c756361834801463992d240-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:21,513 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(327): Store=bac3f6700c756361834801463992d240/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:21,513 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bac3f6700c756361834801463992d240 2024-12-02T16:23:21,514 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bac3f6700c756361834801463992d240 columnFamilyName b 2024-12-02T16:23:21,514 DEBUG [StoreOpener-bac3f6700c756361834801463992d240-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:21,515 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(327): Store=bac3f6700c756361834801463992d240/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:21,515 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bac3f6700c756361834801463992d240 2024-12-02T16:23:21,516 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bac3f6700c756361834801463992d240 columnFamilyName c 2024-12-02T16:23:21,516 DEBUG [StoreOpener-bac3f6700c756361834801463992d240-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:21,516 INFO [StoreOpener-bac3f6700c756361834801463992d240-1 {}] regionserver.HStore(327): Store=bac3f6700c756361834801463992d240/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:21,516 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bac3f6700c756361834801463992d240 2024-12-02T16:23:21,517 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240 2024-12-02T16:23:21,518 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240 2024-12-02T16:23:21,519 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/recovered.edits/0000000000000000026 2024-12-02T16:23:21,542 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:21,544 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/recovered.edits/0000000000000000026 2024-12-02T16:23:21,544 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bac3f6700c756361834801463992d240 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-02T16:23:21,558 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/.tmp/a/5825f887bb344826a3a46e414aa6a0a8 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733156601382/Put/seqid=0 2024-12-02T16:23:21,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741911_1089 (size=5523) 2024-12-02T16:23:21,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741911_1089 (size=5523) 2024-12-02T16:23:21,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741911_1089 (size=5523) 2024-12-02T16:23:21,566 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/.tmp/a/5825f887bb344826a3a46e414aa6a0a8 2024-12-02T16:23:21,588 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/.tmp/b/447a7b4dc6f64adeb26fe8df384d3551 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733156601374/Put/seqid=0 2024-12-02T16:23:21,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741912_1090 (size=5524) 2024-12-02T16:23:21,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741912_1090 (size=5524) 2024-12-02T16:23:21,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741912_1090 (size=5524) 2024-12-02T16:23:21,595 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/.tmp/b/447a7b4dc6f64adeb26fe8df384d3551 2024-12-02T16:23:21,613 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/.tmp/c/22a47541afde499b8efb651fd864fefd is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733156601378/Put/seqid=0 2024-12-02T16:23:21,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741913_1091 (size=5457) 2024-12-02T16:23:21,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741913_1091 (size=5457) 2024-12-02T16:23:21,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741913_1091 (size=5457) 2024-12-02T16:23:21,619 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/.tmp/c/22a47541afde499b8efb651fd864fefd 2024-12-02T16:23:21,625 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/.tmp/a/5825f887bb344826a3a46e414aa6a0a8 as hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/a/5825f887bb344826a3a46e414aa6a0a8 2024-12-02T16:23:21,630 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/a/5825f887bb344826a3a46e414aa6a0a8, entries=7, sequenceid=26, filesize=5.4 K 2024-12-02T16:23:21,631 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/.tmp/b/447a7b4dc6f64adeb26fe8df384d3551 as hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/b/447a7b4dc6f64adeb26fe8df384d3551 2024-12-02T16:23:21,636 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/b/447a7b4dc6f64adeb26fe8df384d3551, entries=7, sequenceid=26, filesize=5.4 K 2024-12-02T16:23:21,638 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/.tmp/c/22a47541afde499b8efb651fd864fefd as hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/c/22a47541afde499b8efb651fd864fefd 2024-12-02T16:23:21,643 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/c/22a47541afde499b8efb651fd864fefd, entries=6, sequenceid=26, filesize=5.3 K 2024-12-02T16:23:21,643 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for bac3f6700c756361834801463992d240 in 99ms, sequenceid=26, compaction requested=false; wal=null 2024-12-02T16:23:21,644 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/recovered.edits/0000000000000000026 2024-12-02T16:23:21,646 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bac3f6700c756361834801463992d240 2024-12-02T16:23:21,646 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bac3f6700c756361834801463992d240 2024-12-02T16:23:21,646 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:21,648 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bac3f6700c756361834801463992d240 2024-12-02T16:23:21,650 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testReplayEditsAfterAbortingFlush/bac3f6700c756361834801463992d240/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-02T16:23:21,651 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bac3f6700c756361834801463992d240; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59434682, jitterRate=-0.1143542230129242}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:21,651 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bac3f6700c756361834801463992d240 2024-12-02T16:23:21,652 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bac3f6700c756361834801463992d240: Running coprocessor pre-open hook at 1733156601510Writing region info on filesystem at 1733156601510Initializing all the Stores at 1733156601511 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156601511Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156601511Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156601511Obtaining lock to block concurrent updates at 1733156601544 (+33 ms)Preparing flush snapshotting stores in bac3f6700c756361834801463992d240 at 1733156601544Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733156601544Flushing stores of testReplayEditsAfterAbortingFlush,,1733156592164.bac3f6700c756361834801463992d240. at 1733156601544Flushing bac3f6700c756361834801463992d240/a: creating writer at 1733156601544Flushing bac3f6700c756361834801463992d240/a: appending metadata at 1733156601558 (+14 ms)Flushing bac3f6700c756361834801463992d240/a: closing flushed file at 1733156601558Flushing bac3f6700c756361834801463992d240/b: creating writer at 1733156601571 (+13 ms)Flushing bac3f6700c756361834801463992d240/b: appending metadata at 1733156601587 (+16 ms)Flushing bac3f6700c756361834801463992d240/b: closing flushed file at 1733156601588 (+1 ms)Flushing bac3f6700c756361834801463992d240/c: creating writer at 1733156601600 (+12 ms)Flushing bac3f6700c756361834801463992d240/c: appending metadata at 1733156601612 (+12 ms)Flushing bac3f6700c756361834801463992d240/c: closing flushed file at 1733156601612Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37316e59: reopening flushed file at 1733156601624 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@135a2852: reopening flushed file at 1733156601630 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@752655ac: reopening flushed file at 1733156601637 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for bac3f6700c756361834801463992d240 in 99ms, sequenceid=26, compaction requested=false; wal=null at 1733156601643 (+6 ms)Cleaning up temporary data from old regions at 1733156601646 (+3 ms)Running coprocessor post-open hooks at 1733156601651 (+5 ms)Region opened successfully at 1733156601652 (+1 ms) 2024-12-02T16:23:21,672 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=414 (was 413) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1520450069-172.17.0.2-1733156567009:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:35590 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:52020 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:51980 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:54106 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:35588 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:54086 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1520450069-172.17.0.2-1733156567009:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1520450069-172.17.0.2-1733156567009:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1180 (was 1116) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=283 (was 282) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5500 (was 5846) 2024-12-02T16:23:21,672 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-12-02T16:23:21,684 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=414, OpenFileDescriptor=1180, MaxFileDescriptor=1048576, SystemLoadAverage=283, ProcessCount=11, AvailableMemoryMB=5500 2024-12-02T16:23:21,684 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-12-02T16:23:21,699 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:23:21,701 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:23:21,701 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T16:23:21,703 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-10293898, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/hregion-10293898, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:21,714 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-10293898/hregion-10293898.1733156601704, exclude list is [], retry=0 2024-12-02T16:23:21,717 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:21,717 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:21,717 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:21,740 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-10293898/hregion-10293898.1733156601704 2024-12-02T16:23:21,740 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:43815:43815)] 2024-12-02T16:23:21,741 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9bb61c5f3037a310ffaabc45c7fd9f7d, NAME => 'testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33443/hbase 2024-12-02T16:23:21,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741915_1093 (size=61) 2024-12-02T16:23:21,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741915_1093 (size=61) 2024-12-02T16:23:21,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741915_1093 (size=61) 2024-12-02T16:23:21,749 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:21,751 INFO [StoreOpener-9bb61c5f3037a310ffaabc45c7fd9f7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:21,752 INFO [StoreOpener-9bb61c5f3037a310ffaabc45c7fd9f7d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bb61c5f3037a310ffaabc45c7fd9f7d columnFamilyName a 2024-12-02T16:23:21,752 DEBUG [StoreOpener-9bb61c5f3037a310ffaabc45c7fd9f7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:21,753 INFO [StoreOpener-9bb61c5f3037a310ffaabc45c7fd9f7d-1 {}] regionserver.HStore(327): Store=9bb61c5f3037a310ffaabc45c7fd9f7d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:21,753 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:21,753 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:21,754 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:21,754 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:21,754 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:21,755 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:21,757 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T16:23:21,758 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9bb61c5f3037a310ffaabc45c7fd9f7d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66109809, jitterRate=-0.01488707959651947}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T16:23:21,758 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9bb61c5f3037a310ffaabc45c7fd9f7d: Writing region info on filesystem at 1733156601749Initializing all the Stores at 1733156601750 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156601750Cleaning up temporary data from old regions at 1733156601754 (+4 ms)Region opened successfully at 1733156601758 (+4 ms) 2024-12-02T16:23:21,759 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9bb61c5f3037a310ffaabc45c7fd9f7d, disabling compactions & flushes 2024-12-02T16:23:21,759 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d. 2024-12-02T16:23:21,759 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d. 2024-12-02T16:23:21,759 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d. after waiting 0 ms 2024-12-02T16:23:21,759 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d. 2024-12-02T16:23:21,759 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d. 2024-12-02T16:23:21,759 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9bb61c5f3037a310ffaabc45c7fd9f7d: Waiting for close lock at 1733156601759Disabling compacts and flushes for region at 1733156601759Disabling writes for close at 1733156601759Writing region close event to WAL at 1733156601759Closed at 1733156601759 2024-12-02T16:23:21,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741914_1092 (size=175) 2024-12-02T16:23:21,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741914_1092 (size=175) 2024-12-02T16:23:21,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741914_1092 (size=175) 2024-12-02T16:23:21,765 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T16:23:21,765 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-10293898:(num 1733156601704) 2024-12-02T16:23:21,765 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:23:21,767 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:21,779 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156601767, exclude list is [], retry=0 2024-12-02T16:23:21,781 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:21,782 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:21,782 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:21,806 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156601767 2024-12-02T16:23:21,806 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:33821:33821)] 2024-12-02T16:23:21,806 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9bb61c5f3037a310ffaabc45c7fd9f7d, NAME => 'testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:21,807 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:21,807 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:21,807 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:21,808 INFO [StoreOpener-9bb61c5f3037a310ffaabc45c7fd9f7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:21,809 INFO [StoreOpener-9bb61c5f3037a310ffaabc45c7fd9f7d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bb61c5f3037a310ffaabc45c7fd9f7d columnFamilyName a 2024-12-02T16:23:21,809 DEBUG [StoreOpener-9bb61c5f3037a310ffaabc45c7fd9f7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:21,809 INFO [StoreOpener-9bb61c5f3037a310ffaabc45c7fd9f7d-1 {}] regionserver.HStore(327): Store=9bb61c5f3037a310ffaabc45c7fd9f7d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:21,809 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:21,810 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:21,811 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:21,811 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:21,811 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:21,813 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:21,813 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9bb61c5f3037a310ffaabc45c7fd9f7d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58771033, jitterRate=-0.12424336373806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T16:23:21,814 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9bb61c5f3037a310ffaabc45c7fd9f7d: Writing region info on filesystem at 1733156601807Initializing all the Stores at 1733156601808 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156601808Cleaning up temporary data from old regions at 1733156601811 (+3 ms)Region opened successfully at 1733156601814 (+3 ms) 2024-12-02T16:23:21,821 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9bb61c5f3037a310ffaabc45c7fd9f7d, disabling compactions & flushes 2024-12-02T16:23:21,821 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d. 2024-12-02T16:23:21,821 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d. 2024-12-02T16:23:21,821 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d. after waiting 0 ms 2024-12-02T16:23:21,822 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d. 2024-12-02T16:23:21,822 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d. 2024-12-02T16:23:21,822 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d. 2024-12-02T16:23:21,822 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9bb61c5f3037a310ffaabc45c7fd9f7d: Waiting for close lock at 1733156601821Disabling compacts and flushes for region at 1733156601821Disabling writes for close at 1733156601821Writing region close event to WAL at 1733156601822 (+1 ms)Closed at 1733156601822 2024-12-02T16:23:21,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741916_1094 (size=1795) 2024-12-02T16:23:21,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741916_1094 (size=1795) 2024-12-02T16:23:21,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741916_1094 (size=1795) 2024-12-02T16:23:21,838 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33443/hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156601767, size=1.8 K (1795bytes) 2024-12-02T16:23:21,838 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33443/hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156601767 2024-12-02T16:23:21,838 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33443/hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156601767 after 0ms 2024-12-02T16:23:21,862 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:21,863 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33443/hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156601767 took 25ms 2024-12-02T16:23:21,865 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33443/hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156601767 so closing down 2024-12-02T16:23:21,865 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T16:23:21,866 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733156601767.temp 2024-12-02T16:23:21,889 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/recovered.edits/0000000000000000003-wal.1733156601767.temp 2024-12-02T16:23:21,890 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T16:23:21,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741917_1095 (size=1795) 2024-12-02T16:23:21,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741917_1095 (size=1795) 2024-12-02T16:23:21,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741917_1095 (size=1795) 2024-12-02T16:23:21,896 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/recovered.edits/0000000000000000003-wal.1733156601767.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-02T16:23:21,897 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/recovered.edits/0000000000000000003-wal.1733156601767.temp to hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/recovered.edits/0000000000000000012 2024-12-02T16:23:21,898 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156601767, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-12-02T16:23:21,898 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33443/hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156601767, journal: Splitting hdfs://localhost:33443/hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156601767, size=1.8 K (1795bytes) at 1733156601838Finishing writing output for hdfs://localhost:33443/hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156601767 so closing down at 1733156601865 (+27 ms)Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/recovered.edits/0000000000000000003-wal.1733156601767.temp at 1733156601889 (+24 ms)3 split writer threads finished at 1733156601890 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/recovered.edits/0000000000000000003-wal.1733156601767.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733156601896 (+6 ms)Rename recovered edits hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/recovered.edits/0000000000000000003-wal.1733156601767.temp to hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/recovered.edits/0000000000000000012 at 1733156601898 (+2 ms)Processed 10 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156601767, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1733156601898 2024-12-02T16:23:21,899 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33443/hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156601767 to hdfs://localhost:33443/hbase/oldWALs/wal.1733156601767 2024-12-02T16:23:21,900 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/recovered.edits/0000000000000000012 2024-12-02T16:23:21,925 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:22,193 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:23:22,195 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:22,205 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156602195, exclude list is [], retry=0 2024-12-02T16:23:22,208 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:22,208 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:22,208 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:22,232 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156602195 2024-12-02T16:23:22,232 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:35029:35029)] 2024-12-02T16:23:22,232 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9bb61c5f3037a310ffaabc45c7fd9f7d, NAME => 'testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:22,232 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:22,232 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,232 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,235 INFO [StoreOpener-9bb61c5f3037a310ffaabc45c7fd9f7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,236 INFO [StoreOpener-9bb61c5f3037a310ffaabc45c7fd9f7d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bb61c5f3037a310ffaabc45c7fd9f7d columnFamilyName a 2024-12-02T16:23:22,237 DEBUG [StoreOpener-9bb61c5f3037a310ffaabc45c7fd9f7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:22,237 INFO [StoreOpener-9bb61c5f3037a310ffaabc45c7fd9f7d-1 {}] regionserver.HStore(327): Store=9bb61c5f3037a310ffaabc45c7fd9f7d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:22,237 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,238 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,239 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,240 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/recovered.edits/0000000000000000012 2024-12-02T16:23:22,269 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:22,270 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/recovered.edits/0000000000000000012 2024-12-02T16:23:22,270 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9bb61c5f3037a310ffaabc45c7fd9f7d 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-02T16:23:22,286 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/.tmp/a/6509bb9d12c940168804295771252d15 is 79, key is testDatalossWhenInputError/a:x0/1733156601814/Put/seqid=0 2024-12-02T16:23:22,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741919_1097 (size=5808) 2024-12-02T16:23:22,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741919_1097 (size=5808) 2024-12-02T16:23:22,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741919_1097 (size=5808) 2024-12-02T16:23:22,293 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/.tmp/a/6509bb9d12c940168804295771252d15 2024-12-02T16:23:22,302 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/.tmp/a/6509bb9d12c940168804295771252d15 as hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/a/6509bb9d12c940168804295771252d15 2024-12-02T16:23:22,309 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/a/6509bb9d12c940168804295771252d15, entries=10, sequenceid=12, filesize=5.7 K 2024-12-02T16:23:22,310 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 9bb61c5f3037a310ffaabc45c7fd9f7d in 40ms, sequenceid=12, compaction requested=false; wal=null 2024-12-02T16:23:22,311 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/recovered.edits/0000000000000000012 2024-12-02T16:23:22,312 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,312 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,314 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,316 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-02T16:23:22,317 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9bb61c5f3037a310ffaabc45c7fd9f7d; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68692201, jitterRate=0.023593559861183167}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T16:23:22,317 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9bb61c5f3037a310ffaabc45c7fd9f7d: Writing region info on filesystem at 1733156602232Initializing all the Stores at 1733156602235 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156602235Obtaining lock to block concurrent updates at 1733156602270 (+35 ms)Preparing flush snapshotting stores in 9bb61c5f3037a310ffaabc45c7fd9f7d at 1733156602270Finished memstore snapshotting testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733156602270Flushing stores of testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d. at 1733156602270Flushing 9bb61c5f3037a310ffaabc45c7fd9f7d/a: creating writer at 1733156602270Flushing 9bb61c5f3037a310ffaabc45c7fd9f7d/a: appending metadata at 1733156602286 (+16 ms)Flushing 9bb61c5f3037a310ffaabc45c7fd9f7d/a: closing flushed file at 1733156602286Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b0af151: reopening flushed file at 1733156602300 (+14 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 9bb61c5f3037a310ffaabc45c7fd9f7d in 40ms, sequenceid=12, compaction requested=false; wal=null at 1733156602310 (+10 ms)Cleaning up temporary data from old regions at 1733156602312 (+2 ms)Region opened successfully at 1733156602317 (+5 ms) 2024-12-02T16:23:22,320 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9bb61c5f3037a310ffaabc45c7fd9f7d, NAME => 'testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:22,320 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733156601700.9bb61c5f3037a310ffaabc45c7fd9f7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:22,320 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,320 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,321 INFO [StoreOpener-9bb61c5f3037a310ffaabc45c7fd9f7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,322 INFO [StoreOpener-9bb61c5f3037a310ffaabc45c7fd9f7d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bb61c5f3037a310ffaabc45c7fd9f7d columnFamilyName a 2024-12-02T16:23:22,322 DEBUG [StoreOpener-9bb61c5f3037a310ffaabc45c7fd9f7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:22,327 DEBUG [StoreOpener-9bb61c5f3037a310ffaabc45c7fd9f7d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/a/6509bb9d12c940168804295771252d15 2024-12-02T16:23:22,327 INFO [StoreOpener-9bb61c5f3037a310ffaabc45c7fd9f7d-1 {}] regionserver.HStore(327): Store=9bb61c5f3037a310ffaabc45c7fd9f7d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:22,327 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,328 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,329 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,330 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,330 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,332 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9bb61c5f3037a310ffaabc45c7fd9f7d 2024-12-02T16:23:22,334 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testDatalossWhenInputError/9bb61c5f3037a310ffaabc45c7fd9f7d/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-02T16:23:22,336 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9bb61c5f3037a310ffaabc45c7fd9f7d; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75345049, jitterRate=0.12272872030735016}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T16:23:22,336 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9bb61c5f3037a310ffaabc45c7fd9f7d: Writing region info on filesystem at 1733156602320Initializing all the Stores at 1733156602321 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156602321Cleaning up temporary data from old regions at 1733156602330 (+9 ms)Region opened successfully at 1733156602336 (+6 ms) 2024-12-02T16:23:22,355 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=424 (was 414) Potentially hanging thread: PacketResponder: BP-1520450069-172.17.0.2-1733156567009:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1520450069-172.17.0.2-1733156567009:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:35670 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:54158 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:35590 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:52020 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:52068 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1520450069-172.17.0.2-1733156567009:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:54086 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1262 (was 1180) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=285 (was 283) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5493 (was 5500) 2024-12-02T16:23:22,356 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-12-02T16:23:22,371 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=424, OpenFileDescriptor=1262, MaxFileDescriptor=1048576, SystemLoadAverage=285, ProcessCount=11, AvailableMemoryMB=5493 2024-12-02T16:23:22,372 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-12-02T16:23:22,388 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:23:22,390 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:23:22,390 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T16:23:22,392 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-11927667, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/hregion-11927667, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:22,403 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-11927667/hregion-11927667.1733156602393, exclude list is [], retry=0 2024-12-02T16:23:22,406 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:22,406 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:22,407 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:22,430 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-11927667/hregion-11927667.1733156602393 2024-12-02T16:23:22,430 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:35029:35029)] 2024-12-02T16:23:22,430 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 91927e7223ba842d04ee5ebf8f79a9cc, NAME => 'testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33443/hbase 2024-12-02T16:23:22,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741921_1099 (size=63) 2024-12-02T16:23:22,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741921_1099 (size=63) 2024-12-02T16:23:22,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741921_1099 (size=63) 2024-12-02T16:23:22,441 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:22,442 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,444 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91927e7223ba842d04ee5ebf8f79a9cc columnFamilyName a 2024-12-02T16:23:22,444 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:22,444 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(327): Store=91927e7223ba842d04ee5ebf8f79a9cc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:22,444 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,445 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91927e7223ba842d04ee5ebf8f79a9cc columnFamilyName b 2024-12-02T16:23:22,446 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:22,446 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(327): Store=91927e7223ba842d04ee5ebf8f79a9cc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:22,446 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,447 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91927e7223ba842d04ee5ebf8f79a9cc columnFamilyName c 2024-12-02T16:23:22,448 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:22,448 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(327): Store=91927e7223ba842d04ee5ebf8f79a9cc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:22,448 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,449 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,449 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,451 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,451 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,451 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:22,453 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,455 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T16:23:22,456 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 91927e7223ba842d04ee5ebf8f79a9cc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75320926, jitterRate=0.12236925959587097}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:22,456 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 91927e7223ba842d04ee5ebf8f79a9cc: Writing region info on filesystem at 1733156602441Initializing all the Stores at 1733156602442 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156602442Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156602442Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156602442Cleaning up temporary data from old regions at 1733156602451 (+9 ms)Region opened successfully at 1733156602456 (+5 ms) 2024-12-02T16:23:22,457 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 91927e7223ba842d04ee5ebf8f79a9cc, disabling compactions & flushes 2024-12-02T16:23:22,457 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc. 2024-12-02T16:23:22,457 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc. 2024-12-02T16:23:22,457 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc. after waiting 0 ms 2024-12-02T16:23:22,457 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc. 2024-12-02T16:23:22,457 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc. 2024-12-02T16:23:22,458 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 91927e7223ba842d04ee5ebf8f79a9cc: Waiting for close lock at 1733156602457Disabling compacts and flushes for region at 1733156602457Disabling writes for close at 1733156602457Writing region close event to WAL at 1733156602457Closed at 1733156602457 2024-12-02T16:23:22,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741920_1098 (size=175) 2024-12-02T16:23:22,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741920_1098 (size=175) 2024-12-02T16:23:22,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741920_1098 (size=175) 2024-12-02T16:23:22,464 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T16:23:22,464 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-11927667:(num 1733156602393) 2024-12-02T16:23:22,464 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:23:22,466 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:22,479 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466, exclude list is [], retry=0 2024-12-02T16:23:22,482 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:22,482 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:22,483 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:22,510 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466 2024-12-02T16:23:22,510 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:35029:35029)] 2024-12-02T16:23:22,510 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 91927e7223ba842d04ee5ebf8f79a9cc, NAME => 'testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:22,511 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:22,511 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,511 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,512 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,513 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91927e7223ba842d04ee5ebf8f79a9cc columnFamilyName a 2024-12-02T16:23:22,513 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:22,514 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(327): Store=91927e7223ba842d04ee5ebf8f79a9cc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:22,514 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,515 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91927e7223ba842d04ee5ebf8f79a9cc columnFamilyName b 2024-12-02T16:23:22,515 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:22,515 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(327): Store=91927e7223ba842d04ee5ebf8f79a9cc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:22,516 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,516 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91927e7223ba842d04ee5ebf8f79a9cc columnFamilyName c 2024-12-02T16:23:22,517 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:22,517 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(327): Store=91927e7223ba842d04ee5ebf8f79a9cc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:22,517 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,518 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,519 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,519 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,519 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,520 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:22,521 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:22,522 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 91927e7223ba842d04ee5ebf8f79a9cc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61388589, jitterRate=-0.0852387398481369}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:22,523 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 91927e7223ba842d04ee5ebf8f79a9cc: Writing region info on filesystem at 1733156602511Initializing all the Stores at 1733156602512 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156602512Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156602512Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156602512Cleaning up temporary data from old regions at 1733156602520 (+8 ms)Region opened successfully at 1733156602523 (+3 ms) 2024-12-02T16:23:22,527 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733156602527/Put/seqid=0 2024-12-02T16:23:22,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741923_1101 (size=4875) 2024-12-02T16:23:22,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741923_1101 (size=4875) 2024-12-02T16:23:22,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741923_1101 (size=4875) 2024-12-02T16:23:22,539 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733156602539/Put/seqid=0 2024-12-02T16:23:22,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741924_1102 (size=4875) 2024-12-02T16:23:22,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741924_1102 (size=4875) 2024-12-02T16:23:22,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741924_1102 (size=4875) 2024-12-02T16:23:22,554 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733156602554/Put/seqid=0 2024-12-02T16:23:22,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741925_1103 (size=4875) 2024-12-02T16:23:22,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741925_1103 (size=4875) 2024-12-02T16:23:22,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741925_1103 (size=4875) 2024-12-02T16:23:22,563 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33443/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 91927e7223ba842d04ee5ebf8f79a9cc/a 2024-12-02T16:23:22,567 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-02T16:23:22,567 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-02T16:23:22,567 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33443/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 91927e7223ba842d04ee5ebf8f79a9cc/a 2024-12-02T16:23:22,571 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-02T16:23:22,571 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-02T16:23:22,571 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33443/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 91927e7223ba842d04ee5ebf8f79a9cc/a 2024-12-02T16:23:22,575 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-02T16:23:22,575 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-02T16:23:22,575 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 91927e7223ba842d04ee5ebf8f79a9cc 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-02T16:23:22,590 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/.tmp/a/393856e957534f01a242fe2e98428010 is 55, key is testCompactedBulkLoadedFiles/a:a/1733156602523/Put/seqid=0 2024-12-02T16:23:22,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741926_1104 (size=5107) 2024-12-02T16:23:22,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741926_1104 (size=5107) 2024-12-02T16:23:22,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741926_1104 (size=5107) 2024-12-02T16:23:22,600 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/.tmp/a/393856e957534f01a242fe2e98428010 2024-12-02T16:23:22,605 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/.tmp/a/393856e957534f01a242fe2e98428010 as hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/393856e957534f01a242fe2e98428010 2024-12-02T16:23:22,611 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/393856e957534f01a242fe2e98428010, entries=1, sequenceid=4, filesize=5.0 K 2024-12-02T16:23:22,612 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 91927e7223ba842d04ee5ebf8f79a9cc in 37ms, sequenceid=4, compaction requested=false 2024-12-02T16:23:22,612 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 91927e7223ba842d04ee5ebf8f79a9cc: 2024-12-02T16:23:22,614 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/31667c973d1940d18e92b5b6b1b4e058_SeqId_4_ 2024-12-02T16:23:22,615 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/aef334a64e6c475caf6b1fe5a6cb3561_SeqId_4_ 2024-12-02T16:23:22,617 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/db4765aa54e04fafbc8bd751711b5340_SeqId_4_ 2024-12-02T16:23:22,618 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33443/hbase/testCompactedBulkLoadedFiles/hfile0 into 91927e7223ba842d04ee5ebf8f79a9cc/a as hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/31667c973d1940d18e92b5b6b1b4e058_SeqId_4_ - updating store file list. 2024-12-02T16:23:22,623 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 31667c973d1940d18e92b5b6b1b4e058_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T16:23:22,624 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/31667c973d1940d18e92b5b6b1b4e058_SeqId_4_ into 91927e7223ba842d04ee5ebf8f79a9cc/a 2024-12-02T16:23:22,624 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33443/hbase/testCompactedBulkLoadedFiles/hfile0 into 91927e7223ba842d04ee5ebf8f79a9cc/a (new location: hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/31667c973d1940d18e92b5b6b1b4e058_SeqId_4_) 2024-12-02T16:23:22,625 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33443/hbase/testCompactedBulkLoadedFiles/hfile1 into 91927e7223ba842d04ee5ebf8f79a9cc/a as hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/aef334a64e6c475caf6b1fe5a6cb3561_SeqId_4_ - updating store file list. 2024-12-02T16:23:22,630 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for aef334a64e6c475caf6b1fe5a6cb3561_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T16:23:22,631 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/aef334a64e6c475caf6b1fe5a6cb3561_SeqId_4_ into 91927e7223ba842d04ee5ebf8f79a9cc/a 2024-12-02T16:23:22,631 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33443/hbase/testCompactedBulkLoadedFiles/hfile1 into 91927e7223ba842d04ee5ebf8f79a9cc/a (new location: hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/aef334a64e6c475caf6b1fe5a6cb3561_SeqId_4_) 2024-12-02T16:23:22,632 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33443/hbase/testCompactedBulkLoadedFiles/hfile2 into 91927e7223ba842d04ee5ebf8f79a9cc/a as hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/db4765aa54e04fafbc8bd751711b5340_SeqId_4_ - updating store file list. 2024-12-02T16:23:22,637 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for db4765aa54e04fafbc8bd751711b5340_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T16:23:22,637 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/db4765aa54e04fafbc8bd751711b5340_SeqId_4_ into 91927e7223ba842d04ee5ebf8f79a9cc/a 2024-12-02T16:23:22,637 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33443/hbase/testCompactedBulkLoadedFiles/hfile2 into 91927e7223ba842d04ee5ebf8f79a9cc/a (new location: hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/db4765aa54e04fafbc8bd751711b5340_SeqId_4_) 2024-12-02T16:23:22,644 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T16:23:22,645 DEBUG [Time-limited test {}] regionserver.HStore(1541): 91927e7223ba842d04ee5ebf8f79a9cc/a is initiating major compaction (all files) 2024-12-02T16:23:22,645 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 91927e7223ba842d04ee5ebf8f79a9cc/a in testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc. 2024-12-02T16:23:22,645 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/393856e957534f01a242fe2e98428010, hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/31667c973d1940d18e92b5b6b1b4e058_SeqId_4_, hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/aef334a64e6c475caf6b1fe5a6cb3561_SeqId_4_, hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/db4765aa54e04fafbc8bd751711b5340_SeqId_4_] into tmpdir=hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/.tmp, totalSize=19.3 K 2024-12-02T16:23:22,646 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 393856e957534f01a242fe2e98428010, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733156602523 2024-12-02T16:23:22,646 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 31667c973d1940d18e92b5b6b1b4e058_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-02T16:23:22,646 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting aef334a64e6c475caf6b1fe5a6cb3561_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-02T16:23:22,647 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting db4765aa54e04fafbc8bd751711b5340_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-02T16:23:22,661 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/.tmp/a/2448bc1034bf4bc28a9b9d785c44215e is 55, key is testCompactedBulkLoadedFiles/a:a/1733156602523/Put/seqid=0 2024-12-02T16:23:22,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741927_1105 (size=6154) 2024-12-02T16:23:22,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741927_1105 (size=6154) 2024-12-02T16:23:22,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741927_1105 (size=6154) 2024-12-02T16:23:22,673 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/.tmp/a/2448bc1034bf4bc28a9b9d785c44215e as hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/2448bc1034bf4bc28a9b9d785c44215e 2024-12-02T16:23:22,678 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 91927e7223ba842d04ee5ebf8f79a9cc/a of 91927e7223ba842d04ee5ebf8f79a9cc into 2448bc1034bf4bc28a9b9d785c44215e(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T16:23:22,679 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 91927e7223ba842d04ee5ebf8f79a9cc: 2024-12-02T16:23:22,679 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-02T16:23:22,679 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-02T16:23:22,710 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33443/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466, size=0 (0bytes) 2024-12-02T16:23:22,710 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33443/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466 might be still open, length is 0 2024-12-02T16:23:22,710 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33443/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466 2024-12-02T16:23:22,711 WARN [IPC Server handler 3 on default port 33443 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-12-02T16:23:22,711 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33443/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466 after 1ms 2024-12-02T16:23:25,656 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:35712 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:33041:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35712 dst: /127.0.0.1:33041 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33041 remote=/127.0.0.1:35712]. Total timeout mills is 60000, 57024 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:25,657 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:54212 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:38643:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54212 dst: /127.0.0.1:38643 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:25,657 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:52106 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:33271:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52106 dst: /127.0.0.1:33271 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:25,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741922_1106 (size=1794) 2024-12-02T16:23:25,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741922_1106 (size=1794) 2024-12-02T16:23:25,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741922_1106 (size=1794) 2024-12-02T16:23:26,712 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33443/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466 after 4002ms 2024-12-02T16:23:26,742 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:26,742 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33443/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466 took 4032ms 2024-12-02T16:23:26,744 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33443/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466; continuing. 2024-12-02T16:23:26,744 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33443/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466 so closing down 2024-12-02T16:23:26,744 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T16:23:26,745 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733156602466.temp 2024-12-02T16:23:26,774 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/recovered.edits/0000000000000000003-wal.1733156602466.temp 2024-12-02T16:23:26,775 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T16:23:26,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741928_1107 (size=800) 2024-12-02T16:23:26,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741928_1107 (size=800) 2024-12-02T16:23:26,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741928_1107 (size=800) 2024-12-02T16:23:26,781 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/recovered.edits/0000000000000000003-wal.1733156602466.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-02T16:23:26,782 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/recovered.edits/0000000000000000003-wal.1733156602466.temp to hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/recovered.edits/0000000000000000008 2024-12-02T16:23:26,782 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 39 ms; skipped=3; WAL=hdfs://localhost:33443/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466, size=0, length=0, corrupted=false, cancelled=false 2024-12-02T16:23:26,782 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33443/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466, journal: Splitting hdfs://localhost:33443/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466, size=0 (0bytes) at 1733156602710Finishing writing output for hdfs://localhost:33443/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466 so closing down at 1733156606744 (+4034 ms)Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/recovered.edits/0000000000000000003-wal.1733156602466.temp at 1733156606774 (+30 ms)3 split writer threads finished at 1733156606775 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/recovered.edits/0000000000000000003-wal.1733156602466.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733156606781 (+6 ms)Rename recovered edits hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/recovered.edits/0000000000000000003-wal.1733156602466.temp to hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/recovered.edits/0000000000000000008 at 1733156606782 (+1 ms)Processed 5 edits across 1 Regions in 39 ms; skipped=3; WAL=hdfs://localhost:33443/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466, size=0, length=0, corrupted=false, cancelled=false at 1733156606782 2024-12-02T16:23:26,784 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33443/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466 to hdfs://localhost:33443/hbase/oldWALs/wal.1733156602466 2024-12-02T16:23:26,784 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/recovered.edits/0000000000000000008 2024-12-02T16:23:26,785 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:23:26,786 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:26,800 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156606786, exclude list is [], retry=0 2024-12-02T16:23:26,802 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:26,802 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:26,802 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:26,825 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156606786 2024-12-02T16:23:26,825 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:43815:43815)] 2024-12-02T16:23:26,826 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 91927e7223ba842d04ee5ebf8f79a9cc, NAME => 'testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:26,826 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:26,826 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:26,826 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:26,827 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:26,828 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91927e7223ba842d04ee5ebf8f79a9cc columnFamilyName a 2024-12-02T16:23:26,828 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:26,837 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/2448bc1034bf4bc28a9b9d785c44215e 2024-12-02T16:23:26,840 DEBUG [StoreFileOpener-91927e7223ba842d04ee5ebf8f79a9cc-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 31667c973d1940d18e92b5b6b1b4e058_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T16:23:26,841 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/31667c973d1940d18e92b5b6b1b4e058_SeqId_4_ 2024-12-02T16:23:26,845 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/393856e957534f01a242fe2e98428010 2024-12-02T16:23:26,848 DEBUG [StoreFileOpener-91927e7223ba842d04ee5ebf8f79a9cc-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for aef334a64e6c475caf6b1fe5a6cb3561_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T16:23:26,849 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/aef334a64e6c475caf6b1fe5a6cb3561_SeqId_4_ 2024-12-02T16:23:26,852 DEBUG [StoreFileOpener-91927e7223ba842d04ee5ebf8f79a9cc-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for db4765aa54e04fafbc8bd751711b5340_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T16:23:26,853 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/db4765aa54e04fafbc8bd751711b5340_SeqId_4_ 2024-12-02T16:23:26,853 WARN [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/31667c973d1940d18e92b5b6b1b4e058_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@56849979 2024-12-02T16:23:26,853 WARN [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/393856e957534f01a242fe2e98428010 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@56849979 2024-12-02T16:23:26,853 WARN [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/aef334a64e6c475caf6b1fe5a6cb3561_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@56849979 2024-12-02T16:23:26,853 WARN [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/db4765aa54e04fafbc8bd751711b5340_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@56849979 2024-12-02T16:23:26,853 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/31667c973d1940d18e92b5b6b1b4e058_SeqId_4_, hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/393856e957534f01a242fe2e98428010, hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/aef334a64e6c475caf6b1fe5a6cb3561_SeqId_4_, hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/db4765aa54e04fafbc8bd751711b5340_SeqId_4_] to archive 2024-12-02T16:23:26,854 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T16:23:26,856 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/31667c973d1940d18e92b5b6b1b4e058_SeqId_4_ to hdfs://localhost:33443/hbase/archive/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/31667c973d1940d18e92b5b6b1b4e058_SeqId_4_ 2024-12-02T16:23:26,857 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/393856e957534f01a242fe2e98428010 to hdfs://localhost:33443/hbase/archive/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/393856e957534f01a242fe2e98428010 2024-12-02T16:23:26,858 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/aef334a64e6c475caf6b1fe5a6cb3561_SeqId_4_ to hdfs://localhost:33443/hbase/archive/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/aef334a64e6c475caf6b1fe5a6cb3561_SeqId_4_ 2024-12-02T16:23:26,859 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/db4765aa54e04fafbc8bd751711b5340_SeqId_4_ to hdfs://localhost:33443/hbase/archive/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/a/db4765aa54e04fafbc8bd751711b5340_SeqId_4_ 2024-12-02T16:23:26,859 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(327): Store=91927e7223ba842d04ee5ebf8f79a9cc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:26,859 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:26,860 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91927e7223ba842d04ee5ebf8f79a9cc columnFamilyName b 2024-12-02T16:23:26,860 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:26,860 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(327): Store=91927e7223ba842d04ee5ebf8f79a9cc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:26,860 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:26,861 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91927e7223ba842d04ee5ebf8f79a9cc columnFamilyName c 2024-12-02T16:23:26,861 DEBUG [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:26,861 INFO [StoreOpener-91927e7223ba842d04ee5ebf8f79a9cc-1 {}] regionserver.HStore(327): Store=91927e7223ba842d04ee5ebf8f79a9cc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:26,861 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:26,862 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:26,863 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:26,864 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/recovered.edits/0000000000000000008 2024-12-02T16:23:26,888 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:26,890 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 91927e7223ba842d04ee5ebf8f79a9cc : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "91927e7223ba842d04ee5ebf8f79a9cc" family_name: "a" compaction_input: "393856e957534f01a242fe2e98428010" compaction_input: "31667c973d1940d18e92b5b6b1b4e058_SeqId_4_" compaction_input: "aef334a64e6c475caf6b1fe5a6cb3561_SeqId_4_" compaction_input: "db4765aa54e04fafbc8bd751711b5340_SeqId_4_" compaction_output: "2448bc1034bf4bc28a9b9d785c44215e" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-02T16:23:26,890 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-02T16:23:26,890 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/recovered.edits/0000000000000000008 2024-12-02T16:23:26,891 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/recovered.edits/0000000000000000008 2024-12-02T16:23:26,892 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:26,892 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:26,893 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:26,894 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 91927e7223ba842d04ee5ebf8f79a9cc 2024-12-02T16:23:26,895 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T16:23:26,896 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testCompactedBulkLoadedFiles/91927e7223ba842d04ee5ebf8f79a9cc/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-02T16:23:26,897 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 91927e7223ba842d04ee5ebf8f79a9cc; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64706707, jitterRate=-0.035794928669929504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:26,898 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 91927e7223ba842d04ee5ebf8f79a9cc: Writing region info on filesystem at 1733156606826Initializing all the Stores at 1733156606827 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156606827Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156606827Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156606827Cleaning up temporary data from old regions at 1733156606892 (+65 ms)Region opened successfully at 1733156606897 (+5 ms) 2024-12-02T16:23:26,900 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 91927e7223ba842d04ee5ebf8f79a9cc, disabling compactions & flushes 2024-12-02T16:23:26,900 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc. 2024-12-02T16:23:26,900 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc. 2024-12-02T16:23:26,900 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc. after waiting 0 ms 2024-12-02T16:23:26,900 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc. 2024-12-02T16:23:26,901 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733156602389.91927e7223ba842d04ee5ebf8f79a9cc. 2024-12-02T16:23:26,901 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 91927e7223ba842d04ee5ebf8f79a9cc: Waiting for close lock at 1733156606900Disabling compacts and flushes for region at 1733156606900Disabling writes for close at 1733156606900Writing region close event to WAL at 1733156606901 (+1 ms)Closed at 1733156606901 2024-12-02T16:23:26,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741929_1108 (size=175) 2024-12-02T16:23:26,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741929_1108 (size=175) 2024-12-02T16:23:26,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741929_1108 (size=175) 2024-12-02T16:23:26,906 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T16:23:26,906 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733156606786) 2024-12-02T16:23:26,919 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=438 (was 424) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:33443 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1685805399) connection to localhost/127.0.0.1:33443 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2059173628_22 at /127.0.0.1:35242 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38415 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2059173628_22 at /127.0.0.1:50328 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2059173628_22 at /127.0.0.1:59990 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1685805399) connection to localhost/127.0.0.1:38415 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1262) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=285 (was 285), ProcessCount=11 (was 11), AvailableMemoryMB=5494 (was 5493) - AvailableMemoryMB LEAK? - 2024-12-02T16:23:26,920 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-02T16:23:26,933 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=438, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=285, ProcessCount=11, AvailableMemoryMB=5492 2024-12-02T16:23:26,933 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-02T16:23:26,946 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:23:26,948 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T16:23:26,948 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T16:23:26,950 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-99717298, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/hregion-99717298, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:26,961 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-99717298/hregion-99717298.1733156606951, exclude list is [], retry=0 2024-12-02T16:23:26,963 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:26,963 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:26,963 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:26,987 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-99717298/hregion-99717298.1733156606951 2024-12-02T16:23:26,987 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:33821:33821)] 2024-12-02T16:23:26,987 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2d2fed4868a03f2a9d2d7f936a5e6d66, NAME => 'testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33443/hbase 2024-12-02T16:23:26,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741931_1110 (size=67) 2024-12-02T16:23:26,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741931_1110 (size=67) 2024-12-02T16:23:26,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741931_1110 (size=67) 2024-12-02T16:23:26,996 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:26,997 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:26,998 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d2fed4868a03f2a9d2d7f936a5e6d66 columnFamilyName a 2024-12-02T16:23:26,999 DEBUG [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:26,999 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(327): Store=2d2fed4868a03f2a9d2d7f936a5e6d66/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:26,999 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,001 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d2fed4868a03f2a9d2d7f936a5e6d66 columnFamilyName b 2024-12-02T16:23:27,001 DEBUG [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:27,001 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(327): Store=2d2fed4868a03f2a9d2d7f936a5e6d66/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:27,001 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,002 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d2fed4868a03f2a9d2d7f936a5e6d66 columnFamilyName c 2024-12-02T16:23:27,002 DEBUG [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:27,003 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(327): Store=2d2fed4868a03f2a9d2d7f936a5e6d66/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:27,003 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,004 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,004 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,005 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,005 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,005 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:27,007 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,009 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T16:23:27,009 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2d2fed4868a03f2a9d2d7f936a5e6d66; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67300976, jitterRate=0.002862691879272461}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:27,010 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2d2fed4868a03f2a9d2d7f936a5e6d66: Writing region info on filesystem at 1733156606996Initializing all the Stores at 1733156606997 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156606997Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156606997Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156606997Cleaning up temporary data from old regions at 1733156607005 (+8 ms)Region opened successfully at 1733156607010 (+5 ms) 2024-12-02T16:23:27,010 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2d2fed4868a03f2a9d2d7f936a5e6d66, disabling compactions & flushes 2024-12-02T16:23:27,010 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. 2024-12-02T16:23:27,010 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. 2024-12-02T16:23:27,010 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. after waiting 0 ms 2024-12-02T16:23:27,010 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. 2024-12-02T16:23:27,010 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. 2024-12-02T16:23:27,010 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2d2fed4868a03f2a9d2d7f936a5e6d66: Waiting for close lock at 1733156607010Disabling compacts and flushes for region at 1733156607010Disabling writes for close at 1733156607010Writing region close event to WAL at 1733156607010Closed at 1733156607010 2024-12-02T16:23:27,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741930_1109 (size=175) 2024-12-02T16:23:27,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741930_1109 (size=175) 2024-12-02T16:23:27,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741930_1109 (size=175) 2024-12-02T16:23:27,015 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T16:23:27,015 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-99717298:(num 1733156606951) 2024-12-02T16:23:27,015 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:23:27,016 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:27,028 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607017, exclude list is [], retry=0 2024-12-02T16:23:27,031 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:27,031 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:27,031 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:27,054 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607017 2024-12-02T16:23:27,054 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:33821:33821)] 2024-12-02T16:23:27,054 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d2fed4868a03f2a9d2d7f936a5e6d66, NAME => 'testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:27,054 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:27,055 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,055 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,056 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,057 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d2fed4868a03f2a9d2d7f936a5e6d66 columnFamilyName a 2024-12-02T16:23:27,057 DEBUG [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:27,057 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(327): Store=2d2fed4868a03f2a9d2d7f936a5e6d66/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:27,058 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,058 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d2fed4868a03f2a9d2d7f936a5e6d66 columnFamilyName b 2024-12-02T16:23:27,058 DEBUG [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:27,059 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(327): Store=2d2fed4868a03f2a9d2d7f936a5e6d66/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:27,059 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,060 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d2fed4868a03f2a9d2d7f936a5e6d66 columnFamilyName c 2024-12-02T16:23:27,060 DEBUG [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:27,060 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(327): Store=2d2fed4868a03f2a9d2d7f936a5e6d66/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:27,060 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,061 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,062 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,063 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,063 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,063 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:27,064 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,065 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2d2fed4868a03f2a9d2d7f936a5e6d66; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66167694, jitterRate=-0.014024525880813599}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:27,065 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2d2fed4868a03f2a9d2d7f936a5e6d66: Writing region info on filesystem at 1733156607055Initializing all the Stores at 1733156607055Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156607056 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156607056Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156607056Cleaning up temporary data from old regions at 1733156607063 (+7 ms)Region opened successfully at 1733156607065 (+2 ms) 2024-12-02T16:23:27,073 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-02T16:23:27,086 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/a/55895f612aa64d5aae16aaeb0ef187da is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733156607065/Put/seqid=0 2024-12-02T16:23:27,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741933_1112 (size=5958) 2024-12-02T16:23:27,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741933_1112 (size=5958) 2024-12-02T16:23:27,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741933_1112 (size=5958) 2024-12-02T16:23:27,093 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/a/55895f612aa64d5aae16aaeb0ef187da 2024-12-02T16:23:27,099 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/a/55895f612aa64d5aae16aaeb0ef187da as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/a/55895f612aa64d5aae16aaeb0ef187da 2024-12-02T16:23:27,104 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/a/55895f612aa64d5aae16aaeb0ef187da, entries=10, sequenceid=13, filesize=5.8 K 2024-12-02T16:23:27,105 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 2d2fed4868a03f2a9d2d7f936a5e6d66 in 32ms, sequenceid=13, compaction requested=false 2024-12-02T16:23:27,106 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2d2fed4868a03f2a9d2d7f936a5e6d66: 2024-12-02T16:23:27,123 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2d2fed4868a03f2a9d2d7f936a5e6d66, disabling compactions & flushes 2024-12-02T16:23:27,123 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. 2024-12-02T16:23:27,123 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. 2024-12-02T16:23:27,123 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. after waiting 0 ms 2024-12-02T16:23:27,123 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. 2024-12-02T16:23:27,124 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. 2024-12-02T16:23:27,124 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. 2024-12-02T16:23:27,125 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2d2fed4868a03f2a9d2d7f936a5e6d66: Waiting for close lock at 1733156607123Disabling compacts and flushes for region at 1733156607123Disabling writes for close at 1733156607123Writing region close event to WAL at 1733156607124 (+1 ms)Closed at 1733156607124 2024-12-02T16:23:27,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741932_1111 (size=6243) 2024-12-02T16:23:27,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741932_1111 (size=6243) 2024-12-02T16:23:27,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741932_1111 (size=6243) 2024-12-02T16:23:27,141 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607017, size=6.1 K (6243bytes) 2024-12-02T16:23:27,141 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607017 2024-12-02T16:23:27,141 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607017 after 0ms 2024-12-02T16:23:27,165 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:27,166 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607017 took 25ms 2024-12-02T16:23:27,167 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607017 so closing down 2024-12-02T16:23:27,168 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T16:23:27,169 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733156607017.temp 2024-12-02T16:23:27,193 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000003-wal.1733156607017.temp 2024-12-02T16:23:27,193 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T16:23:27,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741934_1113 (size=5575) 2024-12-02T16:23:27,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741934_1113 (size=5575) 2024-12-02T16:23:27,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741934_1113 (size=5575) 2024-12-02T16:23:27,199 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000003-wal.1733156607017.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-02T16:23:27,200 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000003-wal.1733156607017.temp to hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000035 2024-12-02T16:23:27,200 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607017, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-12-02T16:23:27,200 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607017, journal: Splitting hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607017, size=6.1 K (6243bytes) at 1733156607141Finishing writing output for hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607017 so closing down at 1733156607168 (+27 ms)Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000003-wal.1733156607017.temp at 1733156607193 (+25 ms)3 split writer threads finished at 1733156607193Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000003-wal.1733156607017.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733156607199 (+6 ms)Rename recovered edits hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000003-wal.1733156607017.temp to hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000035 at 1733156607200 (+1 ms)Processed 32 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607017, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1733156607200 2024-12-02T16:23:27,202 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607017 to hdfs://localhost:33443/hbase/oldWALs/wal.1733156607017 2024-12-02T16:23:27,203 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000035 2024-12-02T16:23:27,203 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:23:27,204 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:27,216 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205, exclude list is [], retry=0 2024-12-02T16:23:27,218 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:27,219 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:27,219 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:27,243 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205 2024-12-02T16:23:27,243 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:35029:35029)] 2024-12-02T16:23:27,243 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d2fed4868a03f2a9d2d7f936a5e6d66, NAME => 'testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66.', STARTKEY => '', ENDKEY => ''} 2024-12-02T16:23:27,243 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:27,244 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,244 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,245 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,246 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d2fed4868a03f2a9d2d7f936a5e6d66 columnFamilyName a 2024-12-02T16:23:27,246 DEBUG [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:27,252 DEBUG [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/a/55895f612aa64d5aae16aaeb0ef187da 2024-12-02T16:23:27,252 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(327): Store=2d2fed4868a03f2a9d2d7f936a5e6d66/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:27,252 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,253 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d2fed4868a03f2a9d2d7f936a5e6d66 columnFamilyName b 2024-12-02T16:23:27,254 DEBUG [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:27,254 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(327): Store=2d2fed4868a03f2a9d2d7f936a5e6d66/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:27,254 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,255 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d2fed4868a03f2a9d2d7f936a5e6d66 columnFamilyName c 2024-12-02T16:23:27,255 DEBUG [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:27,255 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(327): Store=2d2fed4868a03f2a9d2d7f936a5e6d66/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:27,255 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,256 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,257 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,258 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000035 2024-12-02T16:23:27,289 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:27,290 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000035 2024-12-02T16:23:27,290 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-02T16:23:27,304 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/b/2871abafe6a34410b06e6ea7f182553b is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733156607106/Put/seqid=0 2024-12-02T16:23:27,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741936_1115 (size=5958) 2024-12-02T16:23:27,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741936_1115 (size=5958) 2024-12-02T16:23:27,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741936_1115 (size=5958) 2024-12-02T16:23:27,310 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/b/2871abafe6a34410b06e6ea7f182553b 2024-12-02T16:23:27,328 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/c/732b042b54144c82834c8ea23542b7e7 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733156607113/Put/seqid=0 2024-12-02T16:23:27,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741937_1116 (size=5958) 2024-12-02T16:23:27,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741937_1116 (size=5958) 2024-12-02T16:23:27,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741937_1116 (size=5958) 2024-12-02T16:23:27,335 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/c/732b042b54144c82834c8ea23542b7e7 2024-12-02T16:23:27,340 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/b/2871abafe6a34410b06e6ea7f182553b as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/b/2871abafe6a34410b06e6ea7f182553b 2024-12-02T16:23:27,345 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/b/2871abafe6a34410b06e6ea7f182553b, entries=10, sequenceid=35, filesize=5.8 K 2024-12-02T16:23:27,346 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/c/732b042b54144c82834c8ea23542b7e7 as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/c/732b042b54144c82834c8ea23542b7e7 2024-12-02T16:23:27,351 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/c/732b042b54144c82834c8ea23542b7e7, entries=10, sequenceid=35, filesize=5.8 K 2024-12-02T16:23:27,351 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 2d2fed4868a03f2a9d2d7f936a5e6d66 in 61ms, sequenceid=35, compaction requested=false; wal=null 2024-12-02T16:23:27,352 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000035 2024-12-02T16:23:27,353 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,353 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,353 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:27,355 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:27,357 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-02T16:23:27,358 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2d2fed4868a03f2a9d2d7f936a5e6d66; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69861208, jitterRate=0.041013121604919434}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:27,358 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2d2fed4868a03f2a9d2d7f936a5e6d66: Writing region info on filesystem at 1733156607244Initializing all the Stores at 1733156607245 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156607245Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156607245Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156607245Obtaining lock to block concurrent updates at 1733156607290 (+45 ms)Preparing flush snapshotting stores in 2d2fed4868a03f2a9d2d7f936a5e6d66 at 1733156607290Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733156607290Flushing stores of testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. at 1733156607290Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66/b: creating writer at 1733156607290Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66/b: appending metadata at 1733156607304 (+14 ms)Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66/b: closing flushed file at 1733156607304Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66/c: creating writer at 1733156607315 (+11 ms)Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66/c: appending metadata at 1733156607328 (+13 ms)Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66/c: closing flushed file at 1733156607328Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bed8a2d: reopening flushed file at 1733156607339 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bf54a90: reopening flushed file at 1733156607345 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 2d2fed4868a03f2a9d2d7f936a5e6d66 in 61ms, sequenceid=35, compaction requested=false; wal=null at 1733156607351 (+6 ms)Cleaning up temporary data from old regions at 1733156607353 (+2 ms)Region opened successfully at 1733156607358 (+5 ms) 2024-12-02T16:23:27,432 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205, size=0 (0bytes) 2024-12-02T16:23:27,432 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205 might be still open, length is 0 2024-12-02T16:23:27,432 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205 2024-12-02T16:23:27,433 WARN [IPC Server handler 1 on default port 33443 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-12-02T16:23:27,433 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205 after 1ms 2024-12-02T16:23:28,821 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:60072 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:38643:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60072 dst: /127.0.0.1:38643 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38643 remote=/127.0.0.1:60072]. Total timeout mills is 60000, 58568 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:28,821 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:35290 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:33041:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35290 dst: /127.0.0.1:33041 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:28,821 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-326280900_22 at /127.0.0.1:50416 [Receiving block BP-1520450069-172.17.0.2-1733156567009:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:33271:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50416 dst: /127.0.0.1:33271 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:28,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741935_1117 (size=5567) 2024-12-02T16:23:28,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741935_1117 (size=5567) 2024-12-02T16:23:31,010 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-02T16:23:31,010 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-02T16:23:31,010 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-02T16:23:31,010 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-02T16:23:31,435 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205 after 4003ms 2024-12-02T16:23:31,466 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:31,467 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205 took 4034ms 2024-12-02T16:23:31,469 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205; continuing. 2024-12-02T16:23:31,469 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205 so closing down 2024-12-02T16:23:31,469 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T16:23:31,470 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733156607205.temp 2024-12-02T16:23:31,493 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000037-wal.1733156607205.temp 2024-12-02T16:23:31,494 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T16:23:31,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741938_1118 (size=5575) 2024-12-02T16:23:31,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741938_1118 (size=5575) 2024-12-02T16:23:31,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741938_1118 (size=5575) 2024-12-02T16:23:31,501 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000037-wal.1733156607205.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-02T16:23:31,502 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000037-wal.1733156607205.temp to hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000066 2024-12-02T16:23:31,502 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205, size=0, length=0, corrupted=false, cancelled=false 2024-12-02T16:23:31,502 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205, journal: Splitting hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205, size=0 (0bytes) at 1733156607432Finishing writing output for hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205 so closing down at 1733156611469 (+4037 ms)Creating recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000037-wal.1733156607205.temp at 1733156611493 (+24 ms)3 split writer threads finished at 1733156611494 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000037-wal.1733156607205.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733156611501 (+7 ms)Rename recovered edits hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000037-wal.1733156607205.temp to hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000066 at 1733156611502 (+1 ms)Processed 30 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205, size=0, length=0, corrupted=false, cancelled=false at 1733156611502 2024-12-02T16:23:31,503 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205 to hdfs://localhost:33443/hbase/oldWALs/wal.1733156607205 2024-12-02T16:23:31,504 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000066 2024-12-02T16:23:31,504 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T16:23:31,506 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33443/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945, archiveDir=hdfs://localhost:33443/hbase/oldWALs, maxLogs=32 2024-12-02T16:23:31,519 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156611506, exclude list is [], retry=0 2024-12-02T16:23:31,521 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33041,DS-fc4a75e0-1ffd-4a87-994e-31b7f3337aac,DISK] 2024-12-02T16:23:31,521 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38643,DS-3b5bb4e9-dacb-461e-8992-ba1ee005436d,DISK] 2024-12-02T16:23:31,522 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33271,DS-47476b29-867f-4268-be9e-3a4c1c5455c9,DISK] 2024-12-02T16:23:31,545 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156611506 2024-12-02T16:23:31,545 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33821:33821),(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:35029:35029)] 2024-12-02T16:23:31,545 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T16:23:31,547 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:31,547 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d2fed4868a03f2a9d2d7f936a5e6d66 columnFamilyName a 2024-12-02T16:23:31,548 DEBUG [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:31,552 DEBUG [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/a/55895f612aa64d5aae16aaeb0ef187da 2024-12-02T16:23:31,553 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(327): Store=2d2fed4868a03f2a9d2d7f936a5e6d66/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:31,553 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:31,554 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d2fed4868a03f2a9d2d7f936a5e6d66 columnFamilyName b 2024-12-02T16:23:31,554 DEBUG [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:31,559 DEBUG [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/b/2871abafe6a34410b06e6ea7f182553b 2024-12-02T16:23:31,560 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(327): Store=2d2fed4868a03f2a9d2d7f936a5e6d66/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:31,560 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:31,561 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d2fed4868a03f2a9d2d7f936a5e6d66 columnFamilyName c 2024-12-02T16:23:31,561 DEBUG [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T16:23:31,565 DEBUG [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/c/732b042b54144c82834c8ea23542b7e7 2024-12-02T16:23:31,565 INFO [StoreOpener-2d2fed4868a03f2a9d2d7f936a5e6d66-1 {}] regionserver.HStore(327): Store=2d2fed4868a03f2a9d2d7f936a5e6d66/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T16:23:31,566 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:31,567 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:31,568 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:31,568 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000066 2024-12-02T16:23:31,593 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T16:23:31,596 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000066 2024-12-02T16:23:31,597 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-02T16:23:31,609 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/a/a3c27a959c514595b3a8c42d28b1acb4 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733156607364/Put/seqid=0 2024-12-02T16:23:31,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741940_1120 (size=5958) 2024-12-02T16:23:31,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741940_1120 (size=5958) 2024-12-02T16:23:31,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741940_1120 (size=5958) 2024-12-02T16:23:31,616 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/a/a3c27a959c514595b3a8c42d28b1acb4 2024-12-02T16:23:31,635 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/b/c5a02d50ca0747c19b753d355e643352 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733156607371/Put/seqid=0 2024-12-02T16:23:31,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741941_1121 (size=5958) 2024-12-02T16:23:31,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741941_1121 (size=5958) 2024-12-02T16:23:31,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741941_1121 (size=5958) 2024-12-02T16:23:31,641 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/b/c5a02d50ca0747c19b753d355e643352 2024-12-02T16:23:31,659 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/c/072c1e15369d498688ca8a24c5bf3cb2 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733156607380/Put/seqid=0 2024-12-02T16:23:31,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741942_1122 (size=5958) 2024-12-02T16:23:31,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741942_1122 (size=5958) 2024-12-02T16:23:31,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741942_1122 (size=5958) 2024-12-02T16:23:31,666 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/c/072c1e15369d498688ca8a24c5bf3cb2 2024-12-02T16:23:31,670 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/a/a3c27a959c514595b3a8c42d28b1acb4 as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/a/a3c27a959c514595b3a8c42d28b1acb4 2024-12-02T16:23:31,675 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/a/a3c27a959c514595b3a8c42d28b1acb4, entries=10, sequenceid=66, filesize=5.8 K 2024-12-02T16:23:31,676 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/b/c5a02d50ca0747c19b753d355e643352 as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/b/c5a02d50ca0747c19b753d355e643352 2024-12-02T16:23:31,681 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/b/c5a02d50ca0747c19b753d355e643352, entries=10, sequenceid=66, filesize=5.8 K 2024-12-02T16:23:31,682 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/.tmp/c/072c1e15369d498688ca8a24c5bf3cb2 as hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/c/072c1e15369d498688ca8a24c5bf3cb2 2024-12-02T16:23:31,687 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/c/072c1e15369d498688ca8a24c5bf3cb2, entries=10, sequenceid=66, filesize=5.8 K 2024-12-02T16:23:31,687 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 2d2fed4868a03f2a9d2d7f936a5e6d66 in 91ms, sequenceid=66, compaction requested=false; wal=null 2024-12-02T16:23:31,688 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/0000000000000000066 2024-12-02T16:23:31,689 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:31,689 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:31,689 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T16:23:31,690 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2d2fed4868a03f2a9d2d7f936a5e6d66 2024-12-02T16:23:31,692 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/hbase/data/default/testReplayEditsWrittenViaHRegion/2d2fed4868a03f2a9d2d7f936a5e6d66/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-02T16:23:31,693 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2d2fed4868a03f2a9d2d7f936a5e6d66; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69448212, jitterRate=0.03485900163650513}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T16:23:31,693 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2d2fed4868a03f2a9d2d7f936a5e6d66: Writing region info on filesystem at 1733156611546Initializing all the Stores at 1733156611546Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156611546Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156611547 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733156611547Obtaining lock to block concurrent updates at 1733156611597 (+50 ms)Preparing flush snapshotting stores in 2d2fed4868a03f2a9d2d7f936a5e6d66 at 1733156611597Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733156611597Flushing stores of testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. at 1733156611597Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66/a: creating writer at 1733156611597Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66/a: appending metadata at 1733156611609 (+12 ms)Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66/a: closing flushed file at 1733156611609Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66/b: creating writer at 1733156611620 (+11 ms)Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66/b: appending metadata at 1733156611634 (+14 ms)Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66/b: closing flushed file at 1733156611634Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66/c: creating writer at 1733156611645 (+11 ms)Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66/c: appending metadata at 1733156611659 (+14 ms)Flushing 2d2fed4868a03f2a9d2d7f936a5e6d66/c: closing flushed file at 1733156611659Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58cb54f3: reopening flushed file at 1733156611670 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@473a3cd7: reopening flushed file at 1733156611675 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e817748: reopening flushed file at 1733156611681 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 2d2fed4868a03f2a9d2d7f936a5e6d66 in 91ms, sequenceid=66, compaction requested=false; wal=null at 1733156611687 (+6 ms)Cleaning up temporary data from old regions at 1733156611689 (+2 ms)Region opened successfully at 1733156611693 (+4 ms) 2024-12-02T16:23:31,705 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2d2fed4868a03f2a9d2d7f936a5e6d66, disabling compactions & flushes 2024-12-02T16:23:31,705 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. 2024-12-02T16:23:31,706 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. 2024-12-02T16:23:31,706 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. after waiting 0 ms 2024-12-02T16:23:31,706 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. 2024-12-02T16:23:31,707 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733156606947.2d2fed4868a03f2a9d2d7f936a5e6d66. 2024-12-02T16:23:31,707 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2d2fed4868a03f2a9d2d7f936a5e6d66: Waiting for close lock at 1733156611705Disabling compacts and flushes for region at 1733156611705Disabling writes for close at 1733156611706 (+1 ms)Writing region close event to WAL at 1733156611707 (+1 ms)Closed at 1733156611707 2024-12-02T16:23:31,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741939_1119 (size=175) 2024-12-02T16:23:31,710 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156611506 not finished, retry = 0 2024-12-02T16:23:31,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741939_1119 (size=175) 2024-12-02T16:23:31,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741939_1119 (size=175) 2024-12-02T16:23:31,816 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T16:23:31,817 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733156611506) 2024-12-02T16:23:31,837 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=445 (was 438) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:33443 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1849602939_22 at /127.0.0.1:35336 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1685805399) connection to localhost/127.0.0.1:33443 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1849602939_22 at /127.0.0.1:50474 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1849602939_22 at /127.0.0.1:60092 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=286 (was 285) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5463 (was 5492) 2024-12-02T16:23:31,838 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-12-02T16:23:31,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T16:23:31,838 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T16:23:31,838 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T16:23:31,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T16:23:31,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T16:23:31,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T16:23:31,839 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T16:23:31,839 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=818882006, stopped=false 2024-12-02T16:23:31,839 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e5d35c688fae,38905,1733156570684 2024-12-02T16:23:31,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T16:23:31,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T16:23:31,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T16:23:31,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:23:31,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:23:31,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:23:31,883 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T16:23:31,884 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T16:23:31,884 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T16:23:31,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T16:23:31,885 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T16:23:31,885 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T16:23:31,886 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e5d35c688fae,35297,1733156571540' ***** 2024-12-02T16:23:31,886 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T16:23:31,886 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T16:23:31,886 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e5d35c688fae,43901,1733156571586' ***** 2024-12-02T16:23:31,886 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T16:23:31,887 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T16:23:31,887 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T16:23:31,888 INFO [RS:1;e5d35c688fae:35297 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T16:23:31,888 INFO [RS:2;e5d35c688fae:43901 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T16:23:31,888 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T16:23:31,888 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T16:23:31,888 INFO [RS:1;e5d35c688fae:35297 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T16:23:31,888 INFO [RS:2;e5d35c688fae:43901 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T16:23:31,888 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.HRegionServer(3091): Received CLOSE for 01e69ba503cdc68fb29228453655d981 2024-12-02T16:23:31,888 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.HRegionServer(959): stopping server e5d35c688fae,43901,1733156571586 2024-12-02T16:23:31,888 INFO [RS:2;e5d35c688fae:43901 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T16:23:31,888 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.HRegionServer(959): stopping server e5d35c688fae,35297,1733156571540 2024-12-02T16:23:31,888 INFO [RS:2;e5d35c688fae:43901 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;e5d35c688fae:43901. 2024-12-02T16:23:31,888 INFO [RS:1;e5d35c688fae:35297 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T16:23:31,889 DEBUG [RS:2;e5d35c688fae:43901 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T16:23:31,889 INFO [RS:1;e5d35c688fae:35297 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;e5d35c688fae:35297. 2024-12-02T16:23:31,889 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 01e69ba503cdc68fb29228453655d981, disabling compactions & flushes 2024-12-02T16:23:31,889 DEBUG [RS:2;e5d35c688fae:43901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T16:23:31,889 DEBUG [RS:1;e5d35c688fae:35297 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T16:23:31,889 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:31,889 DEBUG [RS:1;e5d35c688fae:35297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T16:23:31,889 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.HRegionServer(976): stopping server e5d35c688fae,43901,1733156571586; all regions closed. 2024-12-02T16:23:31,889 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:31,889 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. after waiting 0 ms 2024-12-02T16:23:31,889 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T16:23:31,889 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T16:23:31,889 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T16:23:31,889 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:31,890 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T16:23:31,890 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-02T16:23:31,890 DEBUG [RS:1;e5d35c688fae:35297 {}] regionserver.HRegionServer(1325): Online Regions={01e69ba503cdc68fb29228453655d981=testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981., 1588230740=hbase:meta,,1.1588230740} 2024-12-02T16:23:31,890 DEBUG [RS:1;e5d35c688fae:35297 {}] regionserver.HRegionServer(1351): Waiting on 01e69ba503cdc68fb29228453655d981, 1588230740 2024-12-02T16:23:31,890 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T16:23:31,890 INFO [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T16:23:31,890 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T16:23:31,890 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T16:23:31,890 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T16:23:31,890 INFO [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-12-02T16:23:31,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741835_1011 (size=175) 2024-12-02T16:23:31,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741835_1011 (size=175) 2024-12-02T16:23:31,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741835_1011 (size=175) 2024-12-02T16:23:31,896 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/.tmp/info/e81635fc1e634a11a740655b5e82fa89 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981./info:regioninfo/1733156590922/Put/seqid=0 2024-12-02T16:23:31,898 DEBUG [RS:2;e5d35c688fae:43901 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/oldWALs 2024-12-02T16:23:31,898 INFO [RS:2;e5d35c688fae:43901 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e5d35c688fae%2C43901%2C1733156571586:(num 1733156573042) 2024-12-02T16:23:31,898 DEBUG [RS:2;e5d35c688fae:43901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T16:23:31,898 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T16:23:31,898 INFO [RS:2;e5d35c688fae:43901 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T16:23:31,898 INFO [RS:2;e5d35c688fae:43901 {}] hbase.ChoreService(370): Chore service for: regionserver/e5d35c688fae:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T16:23:31,898 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T16:23:31,898 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T16:23:31,898 INFO [regionserver/e5d35c688fae:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T16:23:31,898 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T16:23:31,899 INFO [RS:2;e5d35c688fae:43901 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T16:23:31,899 INFO [RS:2;e5d35c688fae:43901 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43901 2024-12-02T16:23:31,899 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/01e69ba503cdc68fb29228453655d981/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-02T16:23:31,900 INFO [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:31,900 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 01e69ba503cdc68fb29228453655d981: Waiting for close lock at 1733156611888Running coprocessor pre-close hooks at 1733156611888Disabling compacts and flushes for region at 1733156611888Disabling writes for close at 1733156611889 (+1 ms)Writing region close event to WAL at 1733156611894 (+5 ms)Running coprocessor post-close hooks at 1733156611900 (+6 ms)Closed at 1733156611900 2024-12-02T16:23:31,900 DEBUG [RS_CLOSE_REGION-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733156586752.01e69ba503cdc68fb29228453655d981. 2024-12-02T16:23:31,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741943_1123 (size=6778) 2024-12-02T16:23:31,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741943_1123 (size=6778) 2024-12-02T16:23:31,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741943_1123 (size=6778) 2024-12-02T16:23:31,905 INFO [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/.tmp/info/e81635fc1e634a11a740655b5e82fa89 2024-12-02T16:23:31,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e5d35c688fae,43901,1733156571586 2024-12-02T16:23:31,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T16:23:31,909 INFO [RS:2;e5d35c688fae:43901 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T16:23:31,911 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/.tmp/info/e81635fc1e634a11a740655b5e82fa89 as hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/info/e81635fc1e634a11a740655b5e82fa89 2024-12-02T16:23:31,915 INFO [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/info/e81635fc1e634a11a740655b5e82fa89, entries=8, sequenceid=23, filesize=6.6 K 2024-12-02T16:23:31,916 INFO [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 26ms, sequenceid=23, compaction requested=false 2024-12-02T16:23:31,916 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T16:23:31,918 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e5d35c688fae,43901,1733156571586] 2024-12-02T16:23:31,921 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-12-02T16:23:31,922 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T16:23:31,922 INFO [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T16:23:31,922 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733156611890Running coprocessor pre-close hooks at 1733156611890Disabling compacts and flushes for region at 1733156611890Disabling writes for close at 1733156611890Obtaining lock to block concurrent updates at 1733156611890Preparing flush snapshotting stores in 1588230740 at 1733156611890Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1733156611891 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733156611891Flushing 1588230740/info: creating writer at 1733156611891Flushing 1588230740/info: appending metadata at 1733156611895 (+4 ms)Flushing 1588230740/info: closing flushed file at 1733156611895Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f056f5d: reopening flushed file at 1733156611910 (+15 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 26ms, sequenceid=23, compaction requested=false at 1733156611916 (+6 ms)Writing region close event to WAL at 1733156611917 (+1 ms)Running coprocessor post-close hooks at 1733156611922 (+5 ms)Closed at 1733156611922 2024-12-02T16:23:31,922 DEBUG [RS_CLOSE_META-regionserver/e5d35c688fae:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T16:23:31,926 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e5d35c688fae,43901,1733156571586 already deleted, retry=false 2024-12-02T16:23:31,926 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e5d35c688fae,43901,1733156571586 expired; onlineServers=1 2024-12-02T16:23:31,950 INFO [regionserver/e5d35c688fae:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T16:23:31,950 INFO [regionserver/e5d35c688fae:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T16:23:32,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T16:23:32,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43901-0x1009bb082170003, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T16:23:32,018 INFO [RS:2;e5d35c688fae:43901 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T16:23:32,018 INFO [RS:2;e5d35c688fae:43901 {}] regionserver.HRegionServer(1031): Exiting; stopping=e5d35c688fae,43901,1733156571586; zookeeper connection closed. 2024-12-02T16:23:32,019 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@333ab5a3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@333ab5a3 2024-12-02T16:23:32,090 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.HRegionServer(976): stopping server e5d35c688fae,35297,1733156571540; all regions closed. 2024-12-02T16:23:32,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741893_1071 (size=2625) 2024-12-02T16:23:32,094 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/WALs/e5d35c688fae,35297,1733156571540/e5d35c688fae%2C35297%2C1733156571540.meta.1733156589957.meta not finished, retry = 0 2024-12-02T16:23:32,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741893_1071 (size=2625) 2024-12-02T16:23:32,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741893_1071 (size=2625) 2024-12-02T16:23:32,200 DEBUG [RS:1;e5d35c688fae:35297 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/oldWALs 2024-12-02T16:23:32,200 INFO [RS:1;e5d35c688fae:35297 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e5d35c688fae%2C35297%2C1733156571540.meta:.meta(num 1733156589957) 2024-12-02T16:23:32,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741833_1009 (size=3621) 2024-12-02T16:23:32,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741833_1009 (size=3621) 2024-12-02T16:23:32,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741833_1009 (size=3621) 2024-12-02T16:23:32,207 DEBUG [RS:1;e5d35c688fae:35297 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/oldWALs 2024-12-02T16:23:32,207 INFO [RS:1;e5d35c688fae:35297 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e5d35c688fae%2C35297%2C1733156571540:(num 1733156573042) 2024-12-02T16:23:32,208 DEBUG [RS:1;e5d35c688fae:35297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T16:23:32,208 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T16:23:32,208 INFO [RS:1;e5d35c688fae:35297 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T16:23:32,208 INFO [RS:1;e5d35c688fae:35297 {}] hbase.ChoreService(370): Chore service for: regionserver/e5d35c688fae:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-02T16:23:32,208 INFO [RS:1;e5d35c688fae:35297 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T16:23:32,208 INFO [regionserver/e5d35c688fae:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T16:23:32,208 INFO [RS:1;e5d35c688fae:35297 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35297 2024-12-02T16:23:32,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e5d35c688fae,35297,1733156571540 2024-12-02T16:23:32,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T16:23:32,241 INFO [RS:1;e5d35c688fae:35297 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T16:23:32,251 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e5d35c688fae,35297,1733156571540] 2024-12-02T16:23:32,259 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e5d35c688fae,35297,1733156571540 already deleted, retry=false 2024-12-02T16:23:32,259 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e5d35c688fae,35297,1733156571540 expired; onlineServers=0 2024-12-02T16:23:32,260 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e5d35c688fae,38905,1733156570684' ***** 2024-12-02T16:23:32,260 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T16:23:32,260 INFO [M:0;e5d35c688fae:38905 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T16:23:32,260 INFO [M:0;e5d35c688fae:38905 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T16:23:32,260 DEBUG [M:0;e5d35c688fae:38905 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T16:23:32,260 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T16:23:32,260 DEBUG [M:0;e5d35c688fae:38905 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T16:23:32,260 DEBUG [master/e5d35c688fae:0:becomeActiveMaster-HFileCleaner.large.0-1733156572711 {}] cleaner.HFileCleaner(306): Exit Thread[master/e5d35c688fae:0:becomeActiveMaster-HFileCleaner.large.0-1733156572711,5,FailOnTimeoutGroup] 2024-12-02T16:23:32,260 DEBUG [master/e5d35c688fae:0:becomeActiveMaster-HFileCleaner.small.0-1733156572712 {}] cleaner.HFileCleaner(306): Exit Thread[master/e5d35c688fae:0:becomeActiveMaster-HFileCleaner.small.0-1733156572712,5,FailOnTimeoutGroup] 2024-12-02T16:23:32,260 INFO [M:0;e5d35c688fae:38905 {}] hbase.ChoreService(370): Chore service for: master/e5d35c688fae:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T16:23:32,260 INFO [M:0;e5d35c688fae:38905 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T16:23:32,260 DEBUG [M:0;e5d35c688fae:38905 {}] master.HMaster(1795): Stopping service threads 2024-12-02T16:23:32,260 INFO [M:0;e5d35c688fae:38905 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T16:23:32,261 INFO [M:0;e5d35c688fae:38905 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T16:23:32,261 INFO [M:0;e5d35c688fae:38905 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T16:23:32,261 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T16:23:32,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T16:23:32,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T16:23:32,268 DEBUG [M:0;e5d35c688fae:38905 {}] zookeeper.ZKUtil(347): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T16:23:32,268 WARN [M:0;e5d35c688fae:38905 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T16:23:32,268 INFO [M:0;e5d35c688fae:38905 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/.lastflushedseqids 2024-12-02T16:23:32,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741944_1124 (size=119) 2024-12-02T16:23:32,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741944_1124 (size=119) 2024-12-02T16:23:32,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741944_1124 (size=119) 2024-12-02T16:23:32,280 INFO [M:0;e5d35c688fae:38905 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T16:23:32,280 INFO [M:0;e5d35c688fae:38905 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T16:23:32,280 DEBUG [M:0;e5d35c688fae:38905 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T16:23:32,280 INFO [M:0;e5d35c688fae:38905 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T16:23:32,280 DEBUG [M:0;e5d35c688fae:38905 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T16:23:32,280 DEBUG [M:0;e5d35c688fae:38905 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T16:23:32,281 DEBUG [M:0;e5d35c688fae:38905 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T16:23:32,281 INFO [M:0;e5d35c688fae:38905 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.38 KB heapSize=102.70 KB 2024-12-02T16:23:32,296 DEBUG [M:0;e5d35c688fae:38905 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/431b46ed7001428ba609412b4b011a8f is 82, key is hbase:meta,,1/info:regioninfo/1733156590195/Put/seqid=0 2024-12-02T16:23:32,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741945_1125 (size=6063) 2024-12-02T16:23:32,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741945_1125 (size=6063) 2024-12-02T16:23:32,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741945_1125 (size=6063) 2024-12-02T16:23:32,302 INFO [M:0;e5d35c688fae:38905 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/431b46ed7001428ba609412b4b011a8f 2024-12-02T16:23:32,321 DEBUG [M:0;e5d35c688fae:38905 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a88050b338104f8494f29a63e707268d is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733156587205/Put/seqid=0 2024-12-02T16:23:32,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741946_1126 (size=7906) 2024-12-02T16:23:32,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741946_1126 (size=7906) 2024-12-02T16:23:32,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741946_1126 (size=7906) 2024-12-02T16:23:32,327 INFO [M:0;e5d35c688fae:38905 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.16 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a88050b338104f8494f29a63e707268d 2024-12-02T16:23:32,331 INFO [M:0;e5d35c688fae:38905 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a88050b338104f8494f29a63e707268d 2024-12-02T16:23:32,345 DEBUG [M:0;e5d35c688fae:38905 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9e40910acbff482c928952ee115602c2 is 69, key is e5d35c688fae,35297,1733156571540/rs:state/1733156572765/Put/seqid=0 2024-12-02T16:23:32,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741947_1127 (size=5445) 2024-12-02T16:23:32,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741947_1127 (size=5445) 2024-12-02T16:23:32,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741947_1127 (size=5445) 2024-12-02T16:23:32,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T16:23:32,351 INFO [RS:1;e5d35c688fae:35297 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T16:23:32,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35297-0x1009bb082170002, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T16:23:32,351 INFO [RS:1;e5d35c688fae:35297 {}] regionserver.HRegionServer(1031): Exiting; stopping=e5d35c688fae,35297,1733156571540; zookeeper connection closed. 2024-12-02T16:23:32,351 INFO [M:0;e5d35c688fae:38905 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9e40910acbff482c928952ee115602c2 2024-12-02T16:23:32,352 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@19cc01c2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@19cc01c2 2024-12-02T16:23:32,352 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-02T16:23:32,355 INFO [M:0;e5d35c688fae:38905 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9e40910acbff482c928952ee115602c2 2024-12-02T16:23:32,356 DEBUG [M:0;e5d35c688fae:38905 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/431b46ed7001428ba609412b4b011a8f as hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/431b46ed7001428ba609412b4b011a8f 2024-12-02T16:23:32,361 INFO [M:0;e5d35c688fae:38905 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/431b46ed7001428ba609412b4b011a8f, entries=14, sequenceid=207, filesize=5.9 K 2024-12-02T16:23:32,362 DEBUG [M:0;e5d35c688fae:38905 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a88050b338104f8494f29a63e707268d as hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a88050b338104f8494f29a63e707268d 2024-12-02T16:23:32,366 INFO [M:0;e5d35c688fae:38905 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a88050b338104f8494f29a63e707268d 2024-12-02T16:23:32,366 INFO [M:0;e5d35c688fae:38905 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a88050b338104f8494f29a63e707268d, entries=21, sequenceid=207, filesize=7.7 K 2024-12-02T16:23:32,367 DEBUG [M:0;e5d35c688fae:38905 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9e40910acbff482c928952ee115602c2 as hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9e40910acbff482c928952ee115602c2 2024-12-02T16:23:32,371 INFO [M:0;e5d35c688fae:38905 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9e40910acbff482c928952ee115602c2 2024-12-02T16:23:32,371 INFO [M:0;e5d35c688fae:38905 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33443/user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9e40910acbff482c928952ee115602c2, entries=3, sequenceid=207, filesize=5.3 K 2024-12-02T16:23:32,372 INFO [M:0;e5d35c688fae:38905 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 91ms, sequenceid=207, compaction requested=false 2024-12-02T16:23:32,374 INFO [M:0;e5d35c688fae:38905 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T16:23:32,374 DEBUG [M:0;e5d35c688fae:38905 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733156612280Disabling compacts and flushes for region at 1733156612280Disabling writes for close at 1733156612281 (+1 ms)Obtaining lock to block concurrent updates at 1733156612281Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733156612281Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85386, getHeapSize=105096, getOffHeapSize=0, getCellsCount=248 at 1733156612281Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733156612282 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733156612282Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733156612296 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733156612296Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733156612306 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733156612320 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733156612320Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733156612331 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733156612345 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733156612345Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1254a23: reopening flushed file at 1733156612355 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6378cad4: reopening flushed file at 1733156612361 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f700c71: reopening flushed file at 1733156612366 (+5 ms)Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 91ms, sequenceid=207, compaction requested=false at 1733156612372 (+6 ms)Writing region close event to WAL at 1733156612374 (+2 ms)Closed at 1733156612374 2024-12-02T16:23:32,376 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/eb8280c2-8e39-ff43-b882-d9e8fc50410b/MasterData/WALs/e5d35c688fae,38905,1733156570684/e5d35c688fae%2C38905%2C1733156570684.1733156571987 not finished, retry = 0 2024-12-02T16:23:32,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38643 is added to blk_1073741830_1006 (size=102150) 2024-12-02T16:23:32,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33271 is added to blk_1073741830_1006 (size=102150) 2024-12-02T16:23:32,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33041 is added to blk_1073741830_1006 (size=102150) 2024-12-02T16:23:32,479 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T16:23:32,479 INFO [M:0;e5d35c688fae:38905 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T16:23:32,479 INFO [M:0;e5d35c688fae:38905 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38905 2024-12-02T16:23:32,480 INFO [M:0;e5d35c688fae:38905 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T16:23:32,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T16:23:32,591 INFO [M:0;e5d35c688fae:38905 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T16:23:32,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38905-0x1009bb082170000, quorum=127.0.0.1:61407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T16:23:32,602 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156601471 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156601471 (inode 16688) Holder DFSClient_NONMAPREDUCE_-326280900_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733156592163/wal.1733156601471 (inode 16688) Holder DFSClient_NONMAPREDUCE_-326280900_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-02T16:23:32,605 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591986 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591986 (inode 16665) Holder DFSClient_NONMAPREDUCE_-326280900_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733156591546/wal.1733156591986 (inode 16665) Holder DFSClient_NONMAPREDUCE_-326280900_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-02T16:23:32,606 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733156582229/wal.1733156582350 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:32,608 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156602195 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156602195 (inode 16714) Holder DFSClient_NONMAPREDUCE_-326280900_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733156601698/wal.1733156602195 (inode 16714) Holder DFSClient_NONMAPREDUCE_-326280900_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-02T16:23:32,608 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733156602388/wal.1733156602466 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:32,608 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733156606945/wal.1733156607205 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:32,609 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733156575234/wal.1733156575321 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T16:23:32,610 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal.1733156575056 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal.1733156575056 (inode 16485) Holder DFSClient_NONMAPREDUCE_-326280900_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733156574700/wal.1733156575056 (inode 16485) Holder DFSClient_NONMAPREDUCE_-326280900_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-02T16:23:32,612 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal.1733156574438 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal.1733156574438 (inode 16462) Holder DFSClient_NONMAPREDUCE_-326280900_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733156574027/wal.1733156574438 (inode 16462) Holder DFSClient_NONMAPREDUCE_-326280900_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-02T16:23:32,617 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@b7f81b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T16:23:32,619 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5d3aa315{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T16:23:32,619 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T16:23:32,619 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@606c1a1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T16:23:32,619 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d7caafc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/hadoop.log.dir/,STOPPED} 2024-12-02T16:23:32,622 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T16:23:32,622 WARN [BP-1520450069-172.17.0.2-1733156567009 heartbeating to localhost/127.0.0.1:33443 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T16:23:32,622 WARN [BP-1520450069-172.17.0.2-1733156567009 heartbeating to localhost/127.0.0.1:33443 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1520450069-172.17.0.2-1733156567009 (Datanode Uuid e03a4d62-a0d6-4c80-951a-c1a27b850867) service to localhost/127.0.0.1:33443 2024-12-02T16:23:32,622 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T16:23:32,623 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data5/current/BP-1520450069-172.17.0.2-1733156567009 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T16:23:32,623 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data6/current/BP-1520450069-172.17.0.2-1733156567009 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T16:23:32,624 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T16:23:32,630 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@565a2c18{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T16:23:32,630 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@311fb016{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T16:23:32,630 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T16:23:32,630 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@540114bd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T16:23:32,630 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3fe9d746{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/hadoop.log.dir/,STOPPED} 2024-12-02T16:23:32,632 WARN [BP-1520450069-172.17.0.2-1733156567009 heartbeating to localhost/127.0.0.1:33443 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T16:23:32,632 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T16:23:32,632 WARN [BP-1520450069-172.17.0.2-1733156567009 heartbeating to localhost/127.0.0.1:33443 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1520450069-172.17.0.2-1733156567009 (Datanode Uuid a5d22e8a-0746-4e0f-af23-884e49b6ed1e) service to localhost/127.0.0.1:33443 2024-12-02T16:23:32,632 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T16:23:32,632 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data3/current/BP-1520450069-172.17.0.2-1733156567009 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T16:23:32,632 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data4/current/BP-1520450069-172.17.0.2-1733156567009 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T16:23:32,633 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T16:23:32,650 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@388bacc3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T16:23:32,651 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6f48fbbe{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T16:23:32,651 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T16:23:32,651 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@63c29ae0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T16:23:32,651 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f494b95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/hadoop.log.dir/,STOPPED} 2024-12-02T16:23:32,652 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T16:23:32,652 WARN [BP-1520450069-172.17.0.2-1733156567009 heartbeating to localhost/127.0.0.1:33443 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T16:23:32,652 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T16:23:32,652 WARN [BP-1520450069-172.17.0.2-1733156567009 heartbeating to localhost/127.0.0.1:33443 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1520450069-172.17.0.2-1733156567009 (Datanode Uuid 6d10331e-a0a5-46c6-bace-e0ef7d82b003) service to localhost/127.0.0.1:33443 2024-12-02T16:23:32,653 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data1/current/BP-1520450069-172.17.0.2-1733156567009 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T16:23:32,653 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/cluster_f147adff-db9f-c126-de48-5563c1e15a38/data/data2/current/BP-1520450069-172.17.0.2-1733156567009 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T16:23:32,654 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T16:23:32,659 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6878e65d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T16:23:32,659 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2f30922e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T16:23:32,659 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T16:23:32,660 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4e578b56{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T16:23:32,660 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6bdd7391{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/11904d0a-f482-ebe6-bc3f-bbedcc84e008/hadoop.log.dir/,STOPPED} 2024-12-02T16:23:32,671 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T16:23:32,717 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down