2024-11-19 18:28:43,381 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-19 18:28:43,392 main DEBUG Took 0.009391 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 18:28:43,393 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 18:28:43,393 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 18:28:43,394 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 18:28:43,395 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 18:28:43,402 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 18:28:43,414 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 18:28:43,415 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 18:28:43,415 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 18:28:43,416 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 18:28:43,416 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 18:28:43,416 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 18:28:43,417 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 18:28:43,417 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 18:28:43,418 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 18:28:43,418 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 18:28:43,419 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 18:28:43,419 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 18:28:43,420 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 18:28:43,420 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 18:28:43,420 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 18:28:43,421 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 18:28:43,421 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 18:28:43,421 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 18:28:43,422 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 18:28:43,422 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 18:28:43,422 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 18:28:43,423 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 18:28:43,423 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 18:28:43,423 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 18:28:43,423 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 18:28:43,424 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 18:28:43,425 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 18:28:43,426 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 18:28:43,428 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 18:28:43,429 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 18:28:43,430 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 18:28:43,430 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 18:28:43,438 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 18:28:43,441 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 18:28:43,442 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 18:28:43,443 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 18:28:43,443 main DEBUG createAppenders(={Console}) 2024-11-19 18:28:43,444 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-11-19 18:28:43,444 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-19 18:28:43,444 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-11-19 18:28:43,445 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 18:28:43,445 main DEBUG OutputStream closed 2024-11-19 18:28:43,445 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 18:28:43,445 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 18:28:43,446 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-11-19 18:28:43,513 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 18:28:43,515 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 18:28:43,516 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 18:28:43,517 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 18:28:43,518 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 18:28:43,518 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 18:28:43,518 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 18:28:43,518 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 18:28:43,519 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 18:28:43,519 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 18:28:43,519 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 18:28:43,520 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 18:28:43,520 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 18:28:43,520 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 18:28:43,520 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 18:28:43,521 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 18:28:43,521 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 18:28:43,522 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 18:28:43,524 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 18:28:43,524 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-11-19 18:28:43,524 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 18:28:43,525 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-11-19T18:28:43,765 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378 2024-11-19 18:28:43,768 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 18:28:43,768 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T18:28:43,777 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-19T18:28:43,802 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T18:28:43,854 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T18:28:43,854 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T18:28:43,866 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-19T18:28:43,880 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8, deleteOnExit=true 2024-11-19T18:28:43,880 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-19T18:28:43,881 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/test.cache.data in system properties and HBase conf 2024-11-19T18:28:43,882 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/hadoop.tmp.dir in system properties and HBase conf 2024-11-19T18:28:43,882 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/hadoop.log.dir in system properties and HBase conf 2024-11-19T18:28:43,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-19T18:28:43,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-19T18:28:43,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-19T18:28:43,964 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T18:28:44,056 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T18:28:44,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-19T18:28:44,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-19T18:28:44,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-19T18:28:44,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T18:28:44,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-19T18:28:44,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-19T18:28:44,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T18:28:44,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T18:28:44,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-19T18:28:44,064 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/nfs.dump.dir in system properties and HBase conf 2024-11-19T18:28:44,064 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/java.io.tmpdir in system properties and HBase conf 2024-11-19T18:28:44,064 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T18:28:44,065 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-19T18:28:44,065 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-19T18:28:44,875 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-19T18:28:44,965 INFO [Time-limited test {}] log.Log(170): Logging initialized @2281ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-19T18:28:45,043 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T18:28:45,104 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T18:28:45,124 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T18:28:45,124 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T18:28:45,125 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T18:28:45,144 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T18:28:45,147 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/hadoop.log.dir/,AVAILABLE} 2024-11-19T18:28:45,148 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T18:28:45,339 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@398d2d49{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/java.io.tmpdir/jetty-localhost-40855-hadoop-hdfs-3_4_1-tests_jar-_-any-8439633417638955525/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T18:28:45,345 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:40855} 2024-11-19T18:28:45,346 INFO [Time-limited test {}] server.Server(415): Started @2662ms 2024-11-19T18:28:45,765 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T18:28:45,772 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T18:28:45,774 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T18:28:45,774 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T18:28:45,775 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-19T18:28:45,775 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/hadoop.log.dir/,AVAILABLE} 2024-11-19T18:28:45,776 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T18:28:45,912 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54387323{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/java.io.tmpdir/jetty-localhost-43101-hadoop-hdfs-3_4_1-tests_jar-_-any-7619189319713580727/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T18:28:45,913 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:43101} 2024-11-19T18:28:45,914 INFO [Time-limited test {}] server.Server(415): Started @3230ms 2024-11-19T18:28:45,971 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T18:28:46,116 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T18:28:46,121 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T18:28:46,122 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T18:28:46,122 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T18:28:46,122 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-19T18:28:46,123 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/hadoop.log.dir/,AVAILABLE} 2024-11-19T18:28:46,124 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T18:28:46,243 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7e9308b2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/java.io.tmpdir/jetty-localhost-46009-hadoop-hdfs-3_4_1-tests_jar-_-any-4546627627398168946/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T18:28:46,243 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:46009} 2024-11-19T18:28:46,244 INFO [Time-limited test {}] server.Server(415): Started @3560ms 2024-11-19T18:28:46,246 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T18:28:46,287 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T18:28:46,291 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T18:28:46,296 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T18:28:46,296 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T18:28:46,297 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T18:28:46,297 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/hadoop.log.dir/,AVAILABLE} 2024-11-19T18:28:46,298 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T18:28:46,426 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3217543c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/java.io.tmpdir/jetty-localhost-46309-hadoop-hdfs-3_4_1-tests_jar-_-any-13159995458343111683/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T18:28:46,427 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:46309} 2024-11-19T18:28:46,428 INFO [Time-limited test {}] server.Server(415): Started @3744ms 2024-11-19T18:28:46,430 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T18:28:46,445 WARN [Thread-109 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/data/data4/current/BP-1034186995-172.17.0.2-1732040924630/current, will proceed with Du for space computation calculation, 2024-11-19T18:28:46,445 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/data/data3/current/BP-1034186995-172.17.0.2-1732040924630/current, will proceed with Du for space computation calculation, 2024-11-19T18:28:46,445 WARN [Thread-110 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/data/data1/current/BP-1034186995-172.17.0.2-1732040924630/current, will proceed with Du for space computation calculation, 2024-11-19T18:28:46,445 WARN [Thread-111 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/data/data2/current/BP-1034186995-172.17.0.2-1732040924630/current, will proceed with Du for space computation calculation, 2024-11-19T18:28:46,498 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T18:28:46,498 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T18:28:46,580 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6eaf3b73e34cfff5 with lease ID 0xbe9ba753d2f31de9: Processing first storage report for DS-e43a1924-a450-49c6-ae8c-1ac234131af5 from datanode DatanodeRegistration(127.0.0.1:33841, datanodeUuid=df847fab-12ca-4b42-956e-a0a03b7ef6d8, infoPort=37769, infoSecurePort=0, ipcPort=46199, storageInfo=lv=-57;cid=testClusterID;nsid=391374742;c=1732040924630) 2024-11-19T18:28:46,580 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/data/data5/current/BP-1034186995-172.17.0.2-1732040924630/current, will proceed with Du for space computation calculation, 2024-11-19T18:28:46,581 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6eaf3b73e34cfff5 with lease ID 0xbe9ba753d2f31de9: from storage DS-e43a1924-a450-49c6-ae8c-1ac234131af5 node DatanodeRegistration(127.0.0.1:33841, datanodeUuid=df847fab-12ca-4b42-956e-a0a03b7ef6d8, infoPort=37769, infoSecurePort=0, ipcPort=46199, storageInfo=lv=-57;cid=testClusterID;nsid=391374742;c=1732040924630), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-19T18:28:46,582 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5777c93f666f36a3 with lease ID 0xbe9ba753d2f31de8: Processing first storage report for DS-1031e5de-a507-4f01-9ea9-9f59931468fa from datanode DatanodeRegistration(127.0.0.1:33471, datanodeUuid=58c9e594-4038-46bc-aaf1-0f2d1735a697, infoPort=33801, infoSecurePort=0, ipcPort=46019, storageInfo=lv=-57;cid=testClusterID;nsid=391374742;c=1732040924630) 2024-11-19T18:28:46,582 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5777c93f666f36a3 with lease ID 0xbe9ba753d2f31de8: from storage DS-1031e5de-a507-4f01-9ea9-9f59931468fa node DatanodeRegistration(127.0.0.1:33471, datanodeUuid=58c9e594-4038-46bc-aaf1-0f2d1735a697, infoPort=33801, infoSecurePort=0, ipcPort=46019, storageInfo=lv=-57;cid=testClusterID;nsid=391374742;c=1732040924630), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-19T18:28:46,582 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6eaf3b73e34cfff5 with lease ID 0xbe9ba753d2f31de9: Processing first storage report for DS-f9a4606e-010a-4213-932c-3491fd4984a9 from datanode DatanodeRegistration(127.0.0.1:33841, datanodeUuid=df847fab-12ca-4b42-956e-a0a03b7ef6d8, infoPort=37769, infoSecurePort=0, ipcPort=46199, storageInfo=lv=-57;cid=testClusterID;nsid=391374742;c=1732040924630) 2024-11-19T18:28:46,583 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6eaf3b73e34cfff5 with lease ID 0xbe9ba753d2f31de9: from storage DS-f9a4606e-010a-4213-932c-3491fd4984a9 node DatanodeRegistration(127.0.0.1:33841, datanodeUuid=df847fab-12ca-4b42-956e-a0a03b7ef6d8, infoPort=37769, infoSecurePort=0, ipcPort=46199, storageInfo=lv=-57;cid=testClusterID;nsid=391374742;c=1732040924630), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-19T18:28:46,583 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5777c93f666f36a3 with lease ID 0xbe9ba753d2f31de8: Processing first storage report for DS-045445af-a428-4154-901d-50272ee18794 from datanode DatanodeRegistration(127.0.0.1:33471, datanodeUuid=58c9e594-4038-46bc-aaf1-0f2d1735a697, infoPort=33801, infoSecurePort=0, ipcPort=46019, storageInfo=lv=-57;cid=testClusterID;nsid=391374742;c=1732040924630) 2024-11-19T18:28:46,583 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5777c93f666f36a3 with lease ID 0xbe9ba753d2f31de8: from storage DS-045445af-a428-4154-901d-50272ee18794 node DatanodeRegistration(127.0.0.1:33471, datanodeUuid=58c9e594-4038-46bc-aaf1-0f2d1735a697, infoPort=33801, infoSecurePort=0, ipcPort=46019, storageInfo=lv=-57;cid=testClusterID;nsid=391374742;c=1732040924630), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T18:28:46,584 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/data/data6/current/BP-1034186995-172.17.0.2-1732040924630/current, will proceed with Du for space computation calculation, 2024-11-19T18:28:46,613 WARN [Thread-119 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T18:28:46,619 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x354f86071c9a0bf3 with lease ID 0xbe9ba753d2f31dea: Processing first storage report for DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc from datanode DatanodeRegistration(127.0.0.1:41941, datanodeUuid=9ab5956c-c134-4391-8dd3-6fd734f8a877, infoPort=40377, infoSecurePort=0, ipcPort=42797, storageInfo=lv=-57;cid=testClusterID;nsid=391374742;c=1732040924630) 2024-11-19T18:28:46,620 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x354f86071c9a0bf3 with lease ID 0xbe9ba753d2f31dea: from storage DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc node DatanodeRegistration(127.0.0.1:41941, datanodeUuid=9ab5956c-c134-4391-8dd3-6fd734f8a877, infoPort=40377, infoSecurePort=0, ipcPort=42797, storageInfo=lv=-57;cid=testClusterID;nsid=391374742;c=1732040924630), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T18:28:46,620 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x354f86071c9a0bf3 with lease ID 0xbe9ba753d2f31dea: Processing first storage report for DS-ea34ebcc-9b6a-4f13-b9a3-658def3e5360 from datanode DatanodeRegistration(127.0.0.1:41941, datanodeUuid=9ab5956c-c134-4391-8dd3-6fd734f8a877, infoPort=40377, infoSecurePort=0, ipcPort=42797, storageInfo=lv=-57;cid=testClusterID;nsid=391374742;c=1732040924630) 2024-11-19T18:28:46,620 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x354f86071c9a0bf3 with lease ID 0xbe9ba753d2f31dea: from storage DS-ea34ebcc-9b6a-4f13-b9a3-658def3e5360 node DatanodeRegistration(127.0.0.1:41941, datanodeUuid=9ab5956c-c134-4391-8dd3-6fd734f8a877, infoPort=40377, infoSecurePort=0, ipcPort=42797, storageInfo=lv=-57;cid=testClusterID;nsid=391374742;c=1732040924630), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T18:28:46,807 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378 2024-11-19T18:28:46,885 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/zookeeper_0, clientPort=52143, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T18:28:46,894 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52143 2024-11-19T18:28:46,904 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T18:28:46,907 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T18:28:47,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741825_1001 (size=7) 2024-11-19T18:28:47,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741825_1001 (size=7) 2024-11-19T18:28:47,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741825_1001 (size=7) 2024-11-19T18:28:47,576 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01 with version=8 2024-11-19T18:28:47,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/hbase-staging 2024-11-19T18:28:47,885 INFO [Time-limited test {}] client.ConnectionUtils(128): master/30db5f576be8:0 server-side Connection retries=45 2024-11-19T18:28:47,896 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T18:28:47,897 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T18:28:47,901 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T18:28:47,901 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T18:28:47,901 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T18:28:48,043 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T18:28:48,104 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T18:28:48,114 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T18:28:48,118 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T18:28:48,148 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 21817 (auto-detected) 2024-11-19T18:28:48,150 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T18:28:48,171 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42887 2024-11-19T18:28:48,193 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42887 connecting to ZooKeeper ensemble=127.0.0.1:52143 2024-11-19T18:28:48,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:428870x0, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T18:28:48,232 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42887-0x101317d11230000 connected 2024-11-19T18:28:48,261 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T18:28:48,265 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T18:28:48,275 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T18:28:48,279 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01, hbase.cluster.distributed=false 2024-11-19T18:28:48,305 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T18:28:48,309 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42887 2024-11-19T18:28:48,310 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42887 2024-11-19T18:28:48,311 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42887 2024-11-19T18:28:48,311 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42887 2024-11-19T18:28:48,312 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42887 2024-11-19T18:28:48,423 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/30db5f576be8:0 server-side Connection retries=45 2024-11-19T18:28:48,425 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T18:28:48,425 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T18:28:48,425 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T18:28:48,426 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T18:28:48,426 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T18:28:48,429 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T18:28:48,431 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T18:28:48,432 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38055 2024-11-19T18:28:48,434 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38055 connecting to ZooKeeper ensemble=127.0.0.1:52143 2024-11-19T18:28:48,435 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T18:28:48,437 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T18:28:48,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:380550x0, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T18:28:48,443 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38055-0x101317d11230001 connected 2024-11-19T18:28:48,443 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T18:28:48,447 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T18:28:48,455 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T18:28:48,457 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T18:28:48,462 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T18:28:48,463 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38055 2024-11-19T18:28:48,468 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38055 2024-11-19T18:28:48,468 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38055 2024-11-19T18:28:48,469 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38055 2024-11-19T18:28:48,469 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38055 2024-11-19T18:28:48,487 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/30db5f576be8:0 server-side Connection retries=45 2024-11-19T18:28:48,487 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T18:28:48,487 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T18:28:48,487 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T18:28:48,488 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T18:28:48,488 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T18:28:48,488 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T18:28:48,488 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T18:28:48,489 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40263 2024-11-19T18:28:48,491 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40263 connecting to ZooKeeper ensemble=127.0.0.1:52143 2024-11-19T18:28:48,492 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T18:28:48,494 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T18:28:48,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:402630x0, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T18:28:48,500 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40263-0x101317d11230002 connected 2024-11-19T18:28:48,501 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T18:28:48,501 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T18:28:48,502 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T18:28:48,503 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T18:28:48,505 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T18:28:48,506 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40263 2024-11-19T18:28:48,506 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40263 2024-11-19T18:28:48,506 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40263 2024-11-19T18:28:48,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40263 2024-11-19T18:28:48,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40263 2024-11-19T18:28:48,526 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/30db5f576be8:0 server-side Connection retries=45 2024-11-19T18:28:48,527 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T18:28:48,527 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T18:28:48,527 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T18:28:48,527 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T18:28:48,527 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T18:28:48,527 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T18:28:48,528 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T18:28:48,529 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41975 2024-11-19T18:28:48,530 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41975 connecting to ZooKeeper ensemble=127.0.0.1:52143 2024-11-19T18:28:48,532 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T18:28:48,534 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T18:28:48,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:419750x0, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T18:28:48,541 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:419750x0, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T18:28:48,541 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41975-0x101317d11230003 connected 2024-11-19T18:28:48,541 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T18:28:48,542 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T18:28:48,543 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41975-0x101317d11230003, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T18:28:48,545 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41975-0x101317d11230003, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T18:28:48,546 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41975 2024-11-19T18:28:48,546 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41975 2024-11-19T18:28:48,547 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41975 2024-11-19T18:28:48,547 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41975 2024-11-19T18:28:48,547 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41975 2024-11-19T18:28:48,564 DEBUG [M:0;30db5f576be8:42887 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;30db5f576be8:42887 2024-11-19T18:28:48,565 INFO [master/30db5f576be8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/30db5f576be8,42887,1732040927686 2024-11-19T18:28:48,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T18:28:48,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41975-0x101317d11230003, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T18:28:48,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T18:28:48,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T18:28:48,577 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/30db5f576be8,42887,1732040927686 2024-11-19T18:28:48,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T18:28:48,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41975-0x101317d11230003, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T18:28:48,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:48,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:48,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41975-0x101317d11230003, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:48,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T18:28:48,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:48,600 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T18:28:48,601 INFO [master/30db5f576be8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/30db5f576be8,42887,1732040927686 from backup master directory 2024-11-19T18:28:48,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/30db5f576be8,42887,1732040927686 2024-11-19T18:28:48,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T18:28:48,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T18:28:48,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41975-0x101317d11230003, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T18:28:48,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T18:28:48,606 WARN [master/30db5f576be8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T18:28:48,606 INFO [master/30db5f576be8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=30db5f576be8,42887,1732040927686 2024-11-19T18:28:48,609 INFO [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T18:28:48,610 INFO [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T18:28:48,674 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/hbase.id] with ID: f2f4b32a-28d6-4df3-bcd2-a308881871fc 2024-11-19T18:28:48,675 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/.tmp/hbase.id 2024-11-19T18:28:48,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741826_1002 (size=42) 2024-11-19T18:28:48,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741826_1002 (size=42) 2024-11-19T18:28:48,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741826_1002 (size=42) 2024-11-19T18:28:48,694 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/.tmp/hbase.id]:[hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/hbase.id] 2024-11-19T18:28:48,741 INFO [master/30db5f576be8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T18:28:48,746 INFO [master/30db5f576be8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T18:28:48,765 INFO [master/30db5f576be8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-19T18:28:48,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:48,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41975-0x101317d11230003, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:48,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:48,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:48,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741827_1003 (size=196) 2024-11-19T18:28:48,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741827_1003 (size=196) 2024-11-19T18:28:48,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741827_1003 (size=196) 2024-11-19T18:28:48,805 INFO [master/30db5f576be8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T18:28:48,806 INFO [master/30db5f576be8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T18:28:48,812 INFO [master/30db5f576be8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:28:48,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741828_1004 (size=1189) 2024-11-19T18:28:48,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741828_1004 (size=1189) 2024-11-19T18:28:48,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741828_1004 (size=1189) 2024-11-19T18:28:48,867 INFO [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store 2024-11-19T18:28:48,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741829_1005 (size=34) 2024-11-19T18:28:48,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741829_1005 (size=34) 2024-11-19T18:28:48,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741829_1005 (size=34) 2024-11-19T18:28:48,896 INFO [master/30db5f576be8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T18:28:48,900 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:28:48,901 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T18:28:48,902 INFO [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T18:28:48,902 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T18:28:48,904 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T18:28:48,904 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T18:28:48,904 INFO [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T18:28:48,906 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732040928901Disabling compacts and flushes for region at 1732040928901Disabling writes for close at 1732040928904 (+3 ms)Writing region close event to WAL at 1732040928904Closed at 1732040928904 2024-11-19T18:28:48,908 WARN [master/30db5f576be8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/.initializing 2024-11-19T18:28:48,909 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/WALs/30db5f576be8,42887,1732040927686 2024-11-19T18:28:48,919 INFO [master/30db5f576be8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T18:28:48,935 INFO [master/30db5f576be8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=30db5f576be8%2C42887%2C1732040927686, suffix=, logDir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/WALs/30db5f576be8,42887,1732040927686, archiveDir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/oldWALs, maxLogs=10 2024-11-19T18:28:48,966 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/WALs/30db5f576be8,42887,1732040927686/30db5f576be8%2C42887%2C1732040927686.1732040928939, exclude list is [], retry=0 2024-11-19T18:28:48,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:28:48,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:48,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:48,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:48,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-19T18:28:49,040 INFO [master/30db5f576be8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/WALs/30db5f576be8,42887,1732040927686/30db5f576be8%2C42887%2C1732040927686.1732040928939 2024-11-19T18:28:49,041 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:33801:33801)] 2024-11-19T18:28:49,042 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:28:49,042 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:28:49,046 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T18:28:49,047 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T18:28:49,088 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T18:28:49,116 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T18:28:49,121 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:49,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T18:28:49,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T18:28:49,127 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T18:28:49,128 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:49,129 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:49,129 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T18:28:49,133 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T18:28:49,133 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:49,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:49,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T18:28:49,137 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T18:28:49,137 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:49,138 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:49,138 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T18:28:49,142 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T18:28:49,143 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T18:28:49,148 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T18:28:49,149 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T18:28:49,153 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T18:28:49,158 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T18:28:49,164 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T18:28:49,165 INFO [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60996471, jitterRate=-0.09108175337314606}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T18:28:49,172 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732040929059Initializing all the Stores at 1732040929062 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732040929062Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040929063 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040929063Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040929063Cleaning up temporary data from old regions at 1732040929149 (+86 ms)Region opened successfully at 1732040929172 (+23 ms) 2024-11-19T18:28:49,173 INFO [master/30db5f576be8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T18:28:49,206 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74dfc7f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=30db5f576be8/172.17.0.2:0 2024-11-19T18:28:49,240 INFO [master/30db5f576be8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T18:28:49,252 INFO [master/30db5f576be8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T18:28:49,253 INFO [master/30db5f576be8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T18:28:49,256 INFO [master/30db5f576be8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T18:28:49,257 INFO [master/30db5f576be8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-19T18:28:49,263 INFO [master/30db5f576be8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-19T18:28:49,263 INFO [master/30db5f576be8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T18:28:49,295 INFO [master/30db5f576be8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T18:28:49,306 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T18:28:49,312 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T18:28:49,315 INFO [master/30db5f576be8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T18:28:49,317 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T18:28:49,318 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T18:28:49,321 INFO [master/30db5f576be8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T18:28:49,324 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T18:28:49,326 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T18:28:49,327 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T18:28:49,329 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T18:28:49,348 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T18:28:49,351 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T18:28:49,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T18:28:49,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T18:28:49,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T18:28:49,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41975-0x101317d11230003, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T18:28:49,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:49,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:49,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41975-0x101317d11230003, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:49,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:49,359 INFO [master/30db5f576be8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=30db5f576be8,42887,1732040927686, sessionid=0x101317d11230000, setting cluster-up flag (Was=false) 2024-11-19T18:28:49,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:49,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:49,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41975-0x101317d11230003, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:49,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:49,379 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T18:28:49,381 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=30db5f576be8,42887,1732040927686 2024-11-19T18:28:49,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:49,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41975-0x101317d11230003, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:49,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:49,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:49,393 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T18:28:49,395 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=30db5f576be8,42887,1732040927686 2024-11-19T18:28:49,401 INFO [master/30db5f576be8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T18:28:49,452 INFO [RS:0;30db5f576be8:38055 {}] regionserver.HRegionServer(746): ClusterId : f2f4b32a-28d6-4df3-bcd2-a308881871fc 2024-11-19T18:28:49,455 DEBUG [RS:0;30db5f576be8:38055 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T18:28:49,460 INFO [RS:2;30db5f576be8:41975 {}] regionserver.HRegionServer(746): ClusterId : f2f4b32a-28d6-4df3-bcd2-a308881871fc 2024-11-19T18:28:49,460 INFO [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(746): ClusterId : f2f4b32a-28d6-4df3-bcd2-a308881871fc 2024-11-19T18:28:49,460 DEBUG [RS:2;30db5f576be8:41975 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T18:28:49,460 DEBUG [RS:1;30db5f576be8:40263 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T18:28:49,461 DEBUG [RS:0;30db5f576be8:38055 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T18:28:49,461 DEBUG [RS:0;30db5f576be8:38055 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T18:28:49,463 DEBUG [RS:1;30db5f576be8:40263 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T18:28:49,463 DEBUG [RS:2;30db5f576be8:41975 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T18:28:49,463 DEBUG [RS:1;30db5f576be8:40263 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T18:28:49,463 DEBUG [RS:2;30db5f576be8:41975 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T18:28:49,466 DEBUG [RS:0;30db5f576be8:38055 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T18:28:49,466 DEBUG [RS:0;30db5f576be8:38055 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aafc2c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=30db5f576be8/172.17.0.2:0 2024-11-19T18:28:49,468 DEBUG [RS:1;30db5f576be8:40263 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T18:28:49,469 DEBUG [RS:2;30db5f576be8:41975 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T18:28:49,469 DEBUG [RS:1;30db5f576be8:40263 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66144c26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=30db5f576be8/172.17.0.2:0 2024-11-19T18:28:49,469 DEBUG [RS:2;30db5f576be8:41975 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679a3bd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=30db5f576be8/172.17.0.2:0 2024-11-19T18:28:49,483 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T18:28:49,485 DEBUG [RS:0;30db5f576be8:38055 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;30db5f576be8:38055 2024-11-19T18:28:49,489 DEBUG [RS:2;30db5f576be8:41975 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;30db5f576be8:41975 2024-11-19T18:28:49,490 INFO [RS:0;30db5f576be8:38055 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T18:28:49,490 INFO [RS:2;30db5f576be8:41975 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T18:28:49,490 INFO [RS:0;30db5f576be8:38055 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T18:28:49,490 INFO [RS:2;30db5f576be8:41975 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T18:28:49,490 DEBUG [RS:0;30db5f576be8:38055 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T18:28:49,490 DEBUG [RS:2;30db5f576be8:41975 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T18:28:49,492 DEBUG [RS:1;30db5f576be8:40263 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;30db5f576be8:40263 2024-11-19T18:28:49,492 INFO [RS:1;30db5f576be8:40263 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T18:28:49,492 INFO [RS:1;30db5f576be8:40263 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T18:28:49,492 DEBUG [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T18:28:49,493 INFO [RS:2;30db5f576be8:41975 {}] regionserver.HRegionServer(2659): reportForDuty to master=30db5f576be8,42887,1732040927686 with port=41975, startcode=1732040928526 2024-11-19T18:28:49,493 INFO [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(2659): reportForDuty to master=30db5f576be8,42887,1732040927686 with port=40263, startcode=1732040928486 2024-11-19T18:28:49,494 INFO [RS:0;30db5f576be8:38055 {}] regionserver.HRegionServer(2659): reportForDuty to master=30db5f576be8,42887,1732040927686 with port=38055, startcode=1732040928383 2024-11-19T18:28:49,495 INFO [master/30db5f576be8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T18:28:49,504 INFO [master/30db5f576be8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T18:28:49,511 DEBUG [RS:2;30db5f576be8:41975 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T18:28:49,511 DEBUG [RS:0;30db5f576be8:38055 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T18:28:49,511 DEBUG [RS:1;30db5f576be8:40263 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T18:28:49,511 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 30db5f576be8,42887,1732040927686 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T18:28:49,520 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/30db5f576be8:0, corePoolSize=5, maxPoolSize=5 2024-11-19T18:28:49,520 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/30db5f576be8:0, corePoolSize=5, maxPoolSize=5 2024-11-19T18:28:49,520 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/30db5f576be8:0, corePoolSize=5, maxPoolSize=5 2024-11-19T18:28:49,521 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/30db5f576be8:0, corePoolSize=5, maxPoolSize=5 2024-11-19T18:28:49,521 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/30db5f576be8:0, corePoolSize=10, maxPoolSize=10 2024-11-19T18:28:49,521 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,521 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/30db5f576be8:0, corePoolSize=2, maxPoolSize=2 2024-11-19T18:28:49,521 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,530 INFO [master/30db5f576be8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732040959530 2024-11-19T18:28:49,532 INFO [master/30db5f576be8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T18:28:49,534 INFO [master/30db5f576be8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T18:28:49,538 INFO [master/30db5f576be8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T18:28:49,539 INFO [master/30db5f576be8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T18:28:49,539 INFO [master/30db5f576be8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T18:28:49,540 INFO [master/30db5f576be8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T18:28:49,544 INFO [master/30db5f576be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,549 INFO [master/30db5f576be8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T18:28:49,550 INFO [master/30db5f576be8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T18:28:49,550 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T18:28:49,551 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T18:28:49,551 INFO [master/30db5f576be8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T18:28:49,554 INFO [master/30db5f576be8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T18:28:49,556 INFO [master/30db5f576be8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T18:28:49,557 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37191, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T18:28:49,557 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37151, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T18:28:49,557 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59609, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T18:28:49,559 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:49,559 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T18:28:49,564 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42887 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T18:28:49,564 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/30db5f576be8:0:becomeActiveMaster-HFileCleaner.large.0-1732040929557,5,FailOnTimeoutGroup] 2024-11-19T18:28:49,567 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/30db5f576be8:0:becomeActiveMaster-HFileCleaner.small.0-1732040929565,5,FailOnTimeoutGroup] 2024-11-19T18:28:49,567 INFO [master/30db5f576be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,567 INFO [master/30db5f576be8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T18:28:49,569 INFO [master/30db5f576be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,569 INFO [master/30db5f576be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,571 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42887 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 30db5f576be8,41975,1732040928526 2024-11-19T18:28:49,574 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42887 {}] master.ServerManager(517): Registering regionserver=30db5f576be8,41975,1732040928526 2024-11-19T18:28:49,583 DEBUG [RS:2;30db5f576be8:41975 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01 2024-11-19T18:28:49,583 DEBUG [RS:2;30db5f576be8:41975 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42267 2024-11-19T18:28:49,583 DEBUG [RS:2;30db5f576be8:41975 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T18:28:49,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T18:28:49,588 DEBUG [RS:2;30db5f576be8:41975 {}] zookeeper.ZKUtil(111): regionserver:41975-0x101317d11230003, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/30db5f576be8,41975,1732040928526 2024-11-19T18:28:49,588 WARN [RS:2;30db5f576be8:41975 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T18:28:49,589 INFO [RS:2;30db5f576be8:41975 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:28:49,589 DEBUG [RS:2;30db5f576be8:41975 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526 2024-11-19T18:28:49,591 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [30db5f576be8,41975,1732040928526] 2024-11-19T18:28:49,600 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42887 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 30db5f576be8,38055,1732040928383 2024-11-19T18:28:49,601 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42887 {}] master.ServerManager(517): Registering regionserver=30db5f576be8,38055,1732040928383 2024-11-19T18:28:49,606 DEBUG [RS:0;30db5f576be8:38055 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01 2024-11-19T18:28:49,606 DEBUG [RS:0;30db5f576be8:38055 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42267 2024-11-19T18:28:49,606 DEBUG [RS:0;30db5f576be8:38055 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T18:28:49,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741831_1007 (size=1321) 2024-11-19T18:28:49,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741831_1007 (size=1321) 2024-11-19T18:28:49,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741831_1007 (size=1321) 2024-11-19T18:28:49,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T18:28:49,613 DEBUG [RS:0;30db5f576be8:38055 {}] zookeeper.ZKUtil(111): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/30db5f576be8,38055,1732040928383 2024-11-19T18:28:49,614 WARN [RS:0;30db5f576be8:38055 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T18:28:49,614 INFO [RS:0;30db5f576be8:38055 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:28:49,614 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [30db5f576be8,38055,1732040928383] 2024-11-19T18:28:49,614 DEBUG [RS:0;30db5f576be8:38055 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,38055,1732040928383 2024-11-19T18:28:49,617 DEBUG [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-19T18:28:49,617 WARN [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-19T18:28:49,636 INFO [RS:0;30db5f576be8:38055 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T18:28:49,636 INFO [RS:2;30db5f576be8:41975 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T18:28:49,653 INFO [RS:0;30db5f576be8:38055 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T18:28:49,653 INFO [RS:2;30db5f576be8:41975 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T18:28:49,659 INFO [RS:2;30db5f576be8:41975 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T18:28:49,659 INFO [RS:0;30db5f576be8:38055 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T18:28:49,659 INFO [RS:0;30db5f576be8:38055 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,659 INFO [RS:2;30db5f576be8:41975 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,660 INFO [RS:0;30db5f576be8:38055 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T18:28:49,660 INFO [RS:2;30db5f576be8:41975 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T18:28:49,667 INFO [RS:0;30db5f576be8:38055 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T18:28:49,667 INFO [RS:2;30db5f576be8:41975 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T18:28:49,668 INFO [RS:0;30db5f576be8:38055 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,668 INFO [RS:2;30db5f576be8:41975 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,669 DEBUG [RS:0;30db5f576be8:38055 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,669 DEBUG [RS:2;30db5f576be8:41975 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,669 DEBUG [RS:0;30db5f576be8:38055 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,669 DEBUG [RS:2;30db5f576be8:41975 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,669 DEBUG [RS:0;30db5f576be8:38055 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,669 DEBUG [RS:2;30db5f576be8:41975 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,669 DEBUG [RS:0;30db5f576be8:38055 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,670 DEBUG [RS:2;30db5f576be8:41975 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,670 DEBUG [RS:0;30db5f576be8:38055 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,670 DEBUG [RS:2;30db5f576be8:41975 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,670 DEBUG [RS:0;30db5f576be8:38055 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/30db5f576be8:0, corePoolSize=2, maxPoolSize=2 2024-11-19T18:28:49,670 DEBUG [RS:0;30db5f576be8:38055 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,670 DEBUG [RS:2;30db5f576be8:41975 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/30db5f576be8:0, corePoolSize=2, maxPoolSize=2 2024-11-19T18:28:49,670 DEBUG [RS:0;30db5f576be8:38055 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,670 DEBUG [RS:2;30db5f576be8:41975 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,670 DEBUG [RS:0;30db5f576be8:38055 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,670 DEBUG [RS:2;30db5f576be8:41975 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,670 DEBUG [RS:0;30db5f576be8:38055 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,671 DEBUG [RS:2;30db5f576be8:41975 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,671 DEBUG [RS:0;30db5f576be8:38055 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,671 DEBUG [RS:2;30db5f576be8:41975 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,671 DEBUG [RS:0;30db5f576be8:38055 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,671 DEBUG [RS:0;30db5f576be8:38055 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/30db5f576be8:0, corePoolSize=3, maxPoolSize=3 2024-11-19T18:28:49,671 DEBUG [RS:2;30db5f576be8:41975 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,671 DEBUG [RS:0;30db5f576be8:38055 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/30db5f576be8:0, corePoolSize=3, maxPoolSize=3 2024-11-19T18:28:49,671 DEBUG [RS:2;30db5f576be8:41975 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,671 DEBUG [RS:2;30db5f576be8:41975 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/30db5f576be8:0, corePoolSize=3, maxPoolSize=3 2024-11-19T18:28:49,671 DEBUG [RS:2;30db5f576be8:41975 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/30db5f576be8:0, corePoolSize=3, maxPoolSize=3 2024-11-19T18:28:49,672 INFO [RS:0;30db5f576be8:38055 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,672 INFO [RS:0;30db5f576be8:38055 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,673 INFO [RS:0;30db5f576be8:38055 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,673 INFO [RS:0;30db5f576be8:38055 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,673 INFO [RS:0;30db5f576be8:38055 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,673 INFO [RS:0;30db5f576be8:38055 {}] hbase.ChoreService(168): Chore ScheduledChore name=30db5f576be8,38055,1732040928383-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T18:28:49,673 INFO [RS:2;30db5f576be8:41975 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,673 INFO [RS:2;30db5f576be8:41975 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,673 INFO [RS:2;30db5f576be8:41975 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,673 INFO [RS:2;30db5f576be8:41975 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,673 INFO [RS:2;30db5f576be8:41975 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,674 INFO [RS:2;30db5f576be8:41975 {}] hbase.ChoreService(168): Chore ScheduledChore name=30db5f576be8,41975,1732040928526-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T18:28:49,693 INFO [RS:2;30db5f576be8:41975 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T18:28:49,693 INFO [RS:0;30db5f576be8:38055 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T18:28:49,696 INFO [RS:0;30db5f576be8:38055 {}] hbase.ChoreService(168): Chore ScheduledChore name=30db5f576be8,38055,1732040928383-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,696 INFO [RS:2;30db5f576be8:41975 {}] hbase.ChoreService(168): Chore ScheduledChore name=30db5f576be8,41975,1732040928526-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,696 INFO [RS:2;30db5f576be8:41975 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,696 INFO [RS:0;30db5f576be8:38055 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,696 INFO [RS:2;30db5f576be8:41975 {}] regionserver.Replication(171): 30db5f576be8,41975,1732040928526 started 2024-11-19T18:28:49,696 INFO [RS:0;30db5f576be8:38055 {}] regionserver.Replication(171): 30db5f576be8,38055,1732040928383 started 2024-11-19T18:28:49,717 INFO [RS:0;30db5f576be8:38055 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,717 INFO [RS:2;30db5f576be8:41975 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,718 INFO [RS:0;30db5f576be8:38055 {}] regionserver.HRegionServer(1482): Serving as 30db5f576be8,38055,1732040928383, RpcServer on 30db5f576be8/172.17.0.2:38055, sessionid=0x101317d11230001 2024-11-19T18:28:49,718 INFO [RS:2;30db5f576be8:41975 {}] regionserver.HRegionServer(1482): Serving as 30db5f576be8,41975,1732040928526, RpcServer on 30db5f576be8/172.17.0.2:41975, sessionid=0x101317d11230003 2024-11-19T18:28:49,718 INFO [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(2659): reportForDuty to master=30db5f576be8,42887,1732040927686 with port=40263, startcode=1732040928486 2024-11-19T18:28:49,719 DEBUG [RS:0;30db5f576be8:38055 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T18:28:49,719 DEBUG [RS:2;30db5f576be8:41975 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T18:28:49,719 DEBUG [RS:0;30db5f576be8:38055 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 30db5f576be8,38055,1732040928383 2024-11-19T18:28:49,719 DEBUG [RS:2;30db5f576be8:41975 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 30db5f576be8,41975,1732040928526 2024-11-19T18:28:49,719 DEBUG [RS:2;30db5f576be8:41975 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '30db5f576be8,41975,1732040928526' 2024-11-19T18:28:49,719 DEBUG [RS:0;30db5f576be8:38055 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '30db5f576be8,38055,1732040928383' 2024-11-19T18:28:49,720 DEBUG [RS:2;30db5f576be8:41975 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T18:28:49,720 DEBUG [RS:0;30db5f576be8:38055 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T18:28:49,720 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42887 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 30db5f576be8,40263,1732040928486 2024-11-19T18:28:49,720 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42887 {}] master.ServerManager(517): Registering regionserver=30db5f576be8,40263,1732040928486 2024-11-19T18:28:49,721 DEBUG [RS:0;30db5f576be8:38055 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T18:28:49,721 DEBUG [RS:2;30db5f576be8:41975 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T18:28:49,723 DEBUG [RS:0;30db5f576be8:38055 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T18:28:49,723 DEBUG [RS:0;30db5f576be8:38055 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T18:28:49,723 DEBUG [RS:0;30db5f576be8:38055 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 30db5f576be8,38055,1732040928383 2024-11-19T18:28:49,723 DEBUG [RS:0;30db5f576be8:38055 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '30db5f576be8,38055,1732040928383' 2024-11-19T18:28:49,723 DEBUG [RS:0;30db5f576be8:38055 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T18:28:49,723 DEBUG [RS:2;30db5f576be8:41975 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T18:28:49,723 DEBUG [RS:2;30db5f576be8:41975 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T18:28:49,724 DEBUG [RS:2;30db5f576be8:41975 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 30db5f576be8,41975,1732040928526 2024-11-19T18:28:49,724 DEBUG [RS:2;30db5f576be8:41975 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '30db5f576be8,41975,1732040928526' 2024-11-19T18:28:49,724 DEBUG [RS:2;30db5f576be8:41975 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T18:28:49,724 DEBUG [RS:0;30db5f576be8:38055 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T18:28:49,725 DEBUG [RS:2;30db5f576be8:41975 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T18:28:49,725 DEBUG [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01 2024-11-19T18:28:49,725 DEBUG [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42267 2024-11-19T18:28:49,725 DEBUG [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T18:28:49,725 DEBUG [RS:0;30db5f576be8:38055 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T18:28:49,725 INFO [RS:0;30db5f576be8:38055 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T18:28:49,725 INFO [RS:0;30db5f576be8:38055 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T18:28:49,725 DEBUG [RS:2;30db5f576be8:41975 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T18:28:49,725 INFO [RS:2;30db5f576be8:41975 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T18:28:49,725 INFO [RS:2;30db5f576be8:41975 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T18:28:49,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T18:28:49,728 DEBUG [RS:1;30db5f576be8:40263 {}] zookeeper.ZKUtil(111): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/30db5f576be8,40263,1732040928486 2024-11-19T18:28:49,728 WARN [RS:1;30db5f576be8:40263 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T18:28:49,728 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [30db5f576be8,40263,1732040928486] 2024-11-19T18:28:49,728 INFO [RS:1;30db5f576be8:40263 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:28:49,728 DEBUG [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,40263,1732040928486 2024-11-19T18:28:49,734 INFO [RS:1;30db5f576be8:40263 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T18:28:49,738 INFO [RS:1;30db5f576be8:40263 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T18:28:49,739 INFO [RS:1;30db5f576be8:40263 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T18:28:49,739 INFO [RS:1;30db5f576be8:40263 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,739 INFO [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T18:28:49,741 INFO [RS:1;30db5f576be8:40263 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T18:28:49,741 INFO [RS:1;30db5f576be8:40263 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,741 DEBUG [RS:1;30db5f576be8:40263 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,742 DEBUG [RS:1;30db5f576be8:40263 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,742 DEBUG [RS:1;30db5f576be8:40263 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,742 DEBUG [RS:1;30db5f576be8:40263 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,742 DEBUG [RS:1;30db5f576be8:40263 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,742 DEBUG [RS:1;30db5f576be8:40263 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/30db5f576be8:0, corePoolSize=2, maxPoolSize=2 2024-11-19T18:28:49,742 DEBUG [RS:1;30db5f576be8:40263 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,742 DEBUG [RS:1;30db5f576be8:40263 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,742 DEBUG [RS:1;30db5f576be8:40263 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,742 DEBUG [RS:1;30db5f576be8:40263 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,742 DEBUG [RS:1;30db5f576be8:40263 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,742 DEBUG [RS:1;30db5f576be8:40263 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/30db5f576be8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T18:28:49,743 DEBUG [RS:1;30db5f576be8:40263 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/30db5f576be8:0, corePoolSize=3, maxPoolSize=3 2024-11-19T18:28:49,743 DEBUG [RS:1;30db5f576be8:40263 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/30db5f576be8:0, corePoolSize=3, maxPoolSize=3 2024-11-19T18:28:49,744 INFO [RS:1;30db5f576be8:40263 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,744 INFO [RS:1;30db5f576be8:40263 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,744 INFO [RS:1;30db5f576be8:40263 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,744 INFO [RS:1;30db5f576be8:40263 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,744 INFO [RS:1;30db5f576be8:40263 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,744 INFO [RS:1;30db5f576be8:40263 {}] hbase.ChoreService(168): Chore ScheduledChore name=30db5f576be8,40263,1732040928486-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T18:28:49,772 INFO [RS:1;30db5f576be8:40263 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T18:28:49,772 INFO [RS:1;30db5f576be8:40263 {}] hbase.ChoreService(168): Chore ScheduledChore name=30db5f576be8,40263,1732040928486-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,772 INFO [RS:1;30db5f576be8:40263 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,772 INFO [RS:1;30db5f576be8:40263 {}] regionserver.Replication(171): 30db5f576be8,40263,1732040928486 started 2024-11-19T18:28:49,794 INFO [RS:1;30db5f576be8:40263 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:49,795 INFO [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(1482): Serving as 30db5f576be8,40263,1732040928486, RpcServer on 30db5f576be8/172.17.0.2:40263, sessionid=0x101317d11230002 2024-11-19T18:28:49,795 DEBUG [RS:1;30db5f576be8:40263 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T18:28:49,795 DEBUG [RS:1;30db5f576be8:40263 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 30db5f576be8,40263,1732040928486 2024-11-19T18:28:49,795 DEBUG [RS:1;30db5f576be8:40263 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '30db5f576be8,40263,1732040928486' 2024-11-19T18:28:49,795 DEBUG [RS:1;30db5f576be8:40263 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T18:28:49,796 DEBUG [RS:1;30db5f576be8:40263 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T18:28:49,797 DEBUG [RS:1;30db5f576be8:40263 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T18:28:49,797 DEBUG [RS:1;30db5f576be8:40263 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T18:28:49,797 DEBUG [RS:1;30db5f576be8:40263 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 30db5f576be8,40263,1732040928486 2024-11-19T18:28:49,797 DEBUG [RS:1;30db5f576be8:40263 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '30db5f576be8,40263,1732040928486' 2024-11-19T18:28:49,797 DEBUG [RS:1;30db5f576be8:40263 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T18:28:49,798 DEBUG [RS:1;30db5f576be8:40263 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T18:28:49,798 DEBUG [RS:1;30db5f576be8:40263 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T18:28:49,798 INFO [RS:1;30db5f576be8:40263 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T18:28:49,798 INFO [RS:1;30db5f576be8:40263 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T18:28:49,830 INFO [RS:2;30db5f576be8:41975 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T18:28:49,830 INFO [RS:0;30db5f576be8:38055 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T18:28:49,834 INFO [RS:2;30db5f576be8:41975 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=30db5f576be8%2C41975%2C1732040928526, suffix=, logDir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526, archiveDir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/oldWALs, maxLogs=32 2024-11-19T18:28:49,834 INFO [RS:0;30db5f576be8:38055 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=30db5f576be8%2C38055%2C1732040928383, suffix=, logDir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,38055,1732040928383, archiveDir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/oldWALs, maxLogs=32 2024-11-19T18:28:49,852 DEBUG [RS:2;30db5f576be8:41975 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526/30db5f576be8%2C41975%2C1732040928526.1732040929838, exclude list is [], retry=0 2024-11-19T18:28:49,853 DEBUG [RS:0;30db5f576be8:38055 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,38055,1732040928383/30db5f576be8%2C38055%2C1732040928383.1732040929838, exclude list is [], retry=0 2024-11-19T18:28:49,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:49,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:49,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:49,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:49,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:49,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:49,864 INFO [RS:2;30db5f576be8:41975 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526/30db5f576be8%2C41975%2C1732040928526.1732040929838 2024-11-19T18:28:49,864 DEBUG [RS:2;30db5f576be8:41975 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:33801:33801)] 2024-11-19T18:28:49,865 INFO [RS:0;30db5f576be8:38055 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,38055,1732040928383/30db5f576be8%2C38055%2C1732040928383.1732040929838 2024-11-19T18:28:49,865 DEBUG [RS:0;30db5f576be8:38055 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:33801:33801)] 2024-11-19T18:28:49,899 INFO [RS:1;30db5f576be8:40263 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T18:28:49,902 INFO [RS:1;30db5f576be8:40263 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=30db5f576be8%2C40263%2C1732040928486, suffix=, logDir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,40263,1732040928486, archiveDir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/oldWALs, maxLogs=32 2024-11-19T18:28:49,920 DEBUG [RS:1;30db5f576be8:40263 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,40263,1732040928486/30db5f576be8%2C40263%2C1732040928486.1732040929904, exclude list is [], retry=0 2024-11-19T18:28:49,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:49,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:49,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:49,931 INFO [RS:1;30db5f576be8:40263 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,40263,1732040928486/30db5f576be8%2C40263%2C1732040928486.1732040929904 2024-11-19T18:28:49,932 DEBUG [RS:1;30db5f576be8:40263 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:33801:33801)] 2024-11-19T18:28:50,011 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T18:28:50,011 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01 2024-11-19T18:28:50,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741835_1011 (size=32) 2024-11-19T18:28:50,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741835_1011 (size=32) 2024-11-19T18:28:50,031 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:28:50,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741835_1011 (size=32) 2024-11-19T18:28:50,034 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T18:28:50,036 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T18:28:50,036 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:50,038 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T18:28:50,038 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T18:28:50,041 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T18:28:50,041 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:50,042 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T18:28:50,042 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T18:28:50,045 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T18:28:50,045 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:50,046 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T18:28:50,046 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T18:28:50,049 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T18:28:50,049 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:50,050 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T18:28:50,051 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T18:28:50,052 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740 2024-11-19T18:28:50,053 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740 2024-11-19T18:28:50,055 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T18:28:50,055 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T18:28:50,056 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T18:28:50,059 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T18:28:50,063 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T18:28:50,064 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75484245, jitterRate=0.12480290234088898}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T18:28:50,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732040930032Initializing all the Stores at 1732040930034 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732040930034Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732040930034Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040930034Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732040930034Cleaning up temporary data from old regions at 1732040930055 (+21 ms)Region opened successfully at 1732040930065 (+10 ms) 2024-11-19T18:28:50,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T18:28:50,066 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T18:28:50,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T18:28:50,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T18:28:50,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T18:28:50,067 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T18:28:50,067 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732040930065Disabling compacts and flushes for region at 1732040930065Disabling writes for close at 1732040930066 (+1 ms)Writing region close event to WAL at 1732040930067 (+1 ms)Closed at 1732040930067 2024-11-19T18:28:50,071 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T18:28:50,071 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T18:28:50,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T18:28:50,086 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T18:28:50,089 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T18:28:50,242 DEBUG [30db5f576be8:42887 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-19T18:28:50,251 DEBUG [30db5f576be8:42887 {}] balancer.BalancerClusterState(204): Hosts are {30db5f576be8=0} racks are {/default-rack=0} 2024-11-19T18:28:50,258 DEBUG [30db5f576be8:42887 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T18:28:50,258 DEBUG [30db5f576be8:42887 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T18:28:50,258 DEBUG [30db5f576be8:42887 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-19T18:28:50,258 DEBUG [30db5f576be8:42887 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T18:28:50,258 DEBUG [30db5f576be8:42887 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T18:28:50,258 DEBUG [30db5f576be8:42887 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-19T18:28:50,258 INFO [30db5f576be8:42887 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T18:28:50,258 INFO [30db5f576be8:42887 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T18:28:50,258 INFO [30db5f576be8:42887 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-19T18:28:50,259 DEBUG [30db5f576be8:42887 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T18:28:50,266 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=30db5f576be8,40263,1732040928486 2024-11-19T18:28:50,275 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 30db5f576be8,40263,1732040928486, state=OPENING 2024-11-19T18:28:50,281 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T18:28:50,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:50,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41975-0x101317d11230003, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:50,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:50,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:28:50,285 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T18:28:50,285 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T18:28:50,285 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T18:28:50,285 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T18:28:50,287 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T18:28:50,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=30db5f576be8,40263,1732040928486}] 2024-11-19T18:28:50,468 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T18:28:50,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47905, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T18:28:50,485 INFO [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T18:28:50,486 INFO [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:28:50,487 INFO [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T18:28:50,490 INFO [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=30db5f576be8%2C40263%2C1732040928486.meta, suffix=.meta, logDir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,40263,1732040928486, archiveDir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/oldWALs, maxLogs=32 2024-11-19T18:28:50,506 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,40263,1732040928486/30db5f576be8%2C40263%2C1732040928486.meta.1732040930492.meta, exclude list is [], retry=0 2024-11-19T18:28:50,510 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:50,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:50,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:50,514 INFO [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,40263,1732040928486/30db5f576be8%2C40263%2C1732040928486.meta.1732040930492.meta 2024-11-19T18:28:50,515 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:33801:33801)] 2024-11-19T18:28:50,515 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:28:50,517 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T18:28:50,519 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T18:28:50,524 INFO [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T18:28:50,529 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T18:28:50,529 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:28:50,530 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T18:28:50,530 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T18:28:50,533 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T18:28:50,535 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T18:28:50,535 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:50,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T18:28:50,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T18:28:50,538 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T18:28:50,538 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:50,539 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T18:28:50,539 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T18:28:50,541 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T18:28:50,541 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:50,542 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T18:28:50,542 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T18:28:50,543 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T18:28:50,543 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:50,544 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T18:28:50,544 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T18:28:50,546 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740 2024-11-19T18:28:50,549 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740 2024-11-19T18:28:50,552 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T18:28:50,552 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T18:28:50,553 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T18:28:50,556 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T18:28:50,558 INFO [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64981072, jitterRate=-0.03170657157897949}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T18:28:50,559 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T18:28:50,561 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732040930530Writing region info on filesystem at 1732040930530Initializing all the Stores at 1732040930532 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732040930533 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732040930533Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040930533Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732040930533Cleaning up temporary data from old regions at 1732040930552 (+19 ms)Running coprocessor post-open hooks at 1732040930559 (+7 ms)Region opened successfully at 1732040930560 (+1 ms) 2024-11-19T18:28:50,569 INFO [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732040930459 2024-11-19T18:28:50,581 DEBUG [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T18:28:50,581 INFO [RS_OPEN_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T18:28:50,583 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=30db5f576be8,40263,1732040928486 2024-11-19T18:28:50,585 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 30db5f576be8,40263,1732040928486, state=OPEN 2024-11-19T18:28:50,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41975-0x101317d11230003, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T18:28:50,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T18:28:50,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T18:28:50,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T18:28:50,588 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T18:28:50,588 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T18:28:50,588 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T18:28:50,588 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T18:28:50,589 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=30db5f576be8,40263,1732040928486 2024-11-19T18:28:50,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T18:28:50,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=30db5f576be8,40263,1732040928486 in 300 msec 2024-11-19T18:28:50,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T18:28:50,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 518 msec 2024-11-19T18:28:50,603 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T18:28:50,603 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T18:28:50,631 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T18:28:50,633 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30db5f576be8,40263,1732040928486, seqNum=-1] 2024-11-19T18:28:50,656 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T18:28:50,659 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43955, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T18:28:50,687 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2530 sec 2024-11-19T18:28:50,687 INFO [master/30db5f576be8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732040930687, completionTime=-1 2024-11-19T18:28:50,690 INFO [master/30db5f576be8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-19T18:28:50,690 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T18:28:50,721 INFO [master/30db5f576be8:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-19T18:28:50,721 INFO [master/30db5f576be8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732040990721 2024-11-19T18:28:50,721 INFO [master/30db5f576be8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732041050721 2024-11-19T18:28:50,721 INFO [master/30db5f576be8:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 31 msec 2024-11-19T18:28:50,723 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-19T18:28:50,730 INFO [master/30db5f576be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30db5f576be8,42887,1732040927686-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:50,731 INFO [master/30db5f576be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30db5f576be8,42887,1732040927686-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:50,731 INFO [master/30db5f576be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30db5f576be8,42887,1732040927686-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:50,732 INFO [master/30db5f576be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-30db5f576be8:42887, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:50,733 INFO [master/30db5f576be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:50,733 INFO [master/30db5f576be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:50,741 DEBUG [master/30db5f576be8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T18:28:50,772 INFO [master/30db5f576be8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.165sec 2024-11-19T18:28:50,774 INFO [master/30db5f576be8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T18:28:50,776 INFO [master/30db5f576be8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T18:28:50,777 INFO [master/30db5f576be8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T18:28:50,778 INFO [master/30db5f576be8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T18:28:50,778 INFO [master/30db5f576be8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T18:28:50,779 INFO [master/30db5f576be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30db5f576be8,42887,1732040927686-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T18:28:50,779 INFO [master/30db5f576be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30db5f576be8,42887,1732040927686-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T18:28:50,784 DEBUG [master/30db5f576be8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T18:28:50,785 INFO [master/30db5f576be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T18:28:50,786 INFO [master/30db5f576be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30db5f576be8,42887,1732040927686-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T18:28:50,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c8ccb42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T18:28:50,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30db5f576be8,42887,-1 for getting cluster id 2024-11-19T18:28:50,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T18:28:50,884 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f2f4b32a-28d6-4df3-bcd2-a308881871fc' 2024-11-19T18:28:50,887 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T18:28:50,887 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f2f4b32a-28d6-4df3-bcd2-a308881871fc" 2024-11-19T18:28:50,887 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b809f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T18:28:50,887 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30db5f576be8,42887,-1] 2024-11-19T18:28:50,890 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T18:28:50,892 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T18:28:50,893 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T18:28:50,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43ce80ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T18:28:50,896 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T18:28:50,903 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30db5f576be8,40263,1732040928486, seqNum=-1] 2024-11-19T18:28:50,903 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T18:28:50,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T18:28:50,930 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=30db5f576be8,42887,1732040927686 2024-11-19T18:28:50,930 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:42267/hbase 2024-11-19T18:28:50,946 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=364, OpenFileDescriptor=613, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=7547 2024-11-19T18:28:50,966 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:28:50,969 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:28:50,970 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T18:28:50,975 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-02479835, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/hregion-02479835, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:28:51,000 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-02479835/hregion-02479835.1732040930977, exclude list is [], retry=0 2024-11-19T18:28:51,005 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:51,005 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:51,006 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:51,009 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-02479835/hregion-02479835.1732040930977 2024-11-19T18:28:51,013 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:40377:40377)] 2024-11-19T18:28:51,013 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d68d927fde8ce9f3e49247d9ae93404f, NAME => 'testReplayEditsWrittenIntoWAL,,1732040930967.d68d927fde8ce9f3e49247d9ae93404f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42267/hbase 2024-11-19T18:28:51,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741838_1014 (size=64) 2024-11-19T18:28:51,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741838_1014 (size=64) 2024-11-19T18:28:51,029 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732040930967.d68d927fde8ce9f3e49247d9ae93404f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:28:51,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741838_1014 (size=64) 2024-11-19T18:28:51,032 INFO [StoreOpener-d68d927fde8ce9f3e49247d9ae93404f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d68d927fde8ce9f3e49247d9ae93404f 2024-11-19T18:28:51,034 INFO [StoreOpener-d68d927fde8ce9f3e49247d9ae93404f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d68d927fde8ce9f3e49247d9ae93404f columnFamilyName a 2024-11-19T18:28:51,034 DEBUG [StoreOpener-d68d927fde8ce9f3e49247d9ae93404f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:51,035 INFO [StoreOpener-d68d927fde8ce9f3e49247d9ae93404f-1 {}] regionserver.HStore(327): Store=d68d927fde8ce9f3e49247d9ae93404f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:51,036 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d68d927fde8ce9f3e49247d9ae93404f 2024-11-19T18:28:51,037 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f 2024-11-19T18:28:51,038 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f 2024-11-19T18:28:51,039 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d68d927fde8ce9f3e49247d9ae93404f 2024-11-19T18:28:51,039 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d68d927fde8ce9f3e49247d9ae93404f 2024-11-19T18:28:51,042 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d68d927fde8ce9f3e49247d9ae93404f 2024-11-19T18:28:51,046 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T18:28:51,046 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d68d927fde8ce9f3e49247d9ae93404f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61480055, jitterRate=-0.08387579023838043}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T18:28:51,048 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d68d927fde8ce9f3e49247d9ae93404f: Writing region info on filesystem at 1732040931029Initializing all the Stores at 1732040931031 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040931031Cleaning up temporary data from old regions at 1732040931039 (+8 ms)Region opened successfully at 1732040931048 (+9 ms) 2024-11-19T18:28:51,048 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d68d927fde8ce9f3e49247d9ae93404f, disabling compactions & flushes 2024-11-19T18:28:51,048 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732040930967.d68d927fde8ce9f3e49247d9ae93404f. 2024-11-19T18:28:51,048 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732040930967.d68d927fde8ce9f3e49247d9ae93404f. 2024-11-19T18:28:51,048 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732040930967.d68d927fde8ce9f3e49247d9ae93404f. after waiting 0 ms 2024-11-19T18:28:51,048 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732040930967.d68d927fde8ce9f3e49247d9ae93404f. 2024-11-19T18:28:51,049 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732040930967.d68d927fde8ce9f3e49247d9ae93404f. 2024-11-19T18:28:51,049 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d68d927fde8ce9f3e49247d9ae93404f: Waiting for close lock at 1732040931048Disabling compacts and flushes for region at 1732040931048Disabling writes for close at 1732040931048Writing region close event to WAL at 1732040931048Closed at 1732040931048 2024-11-19T18:28:51,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741837_1013 (size=93) 2024-11-19T18:28:51,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741837_1013 (size=93) 2024-11-19T18:28:51,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741837_1013 (size=93) 2024-11-19T18:28:51,062 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-19T18:28:51,062 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-02479835:(num 1732040930977) 2024-11-19T18:28:51,064 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-19T18:28:51,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741839_1015 (size=369) 2024-11-19T18:28:51,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741839_1015 (size=369) 2024-11-19T18:28:51,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741839_1015 (size=369) 2024-11-19T18:28:51,085 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-19T18:28:51,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741840_1016 (size=231) 2024-11-19T18:28:51,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741840_1016 (size=231) 2024-11-19T18:28:51,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741840_1016 (size=231) 2024-11-19T18:28:51,127 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-1, size=369 (369bytes) 2024-11-19T18:28:51,128 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-19T18:28:51,128 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-19T18:28:51,129 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-1 2024-11-19T18:28:51,135 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-1 after 4ms 2024-11-19T18:28:51,143 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-1 took 18ms 2024-11-19T18:28:51,148 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-1 so closing down 2024-11-19T18:28:51,151 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-19T18:28:51,153 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/0000000000000000001-wal-1.temp 2024-11-19T18:28:51,154 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T18:28:51,154 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T18:28:51,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741841_1017 (size=369) 2024-11-19T18:28:51,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741841_1017 (size=369) 2024-11-19T18:28:51,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741841_1017 (size=369) 2024-11-19T18:28:51,168 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-19T18:28:51,170 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/0000000000000000002 2024-11-19T18:28:51,174 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-19T18:28:51,174 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-1, journal: Splitting hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-1, size=369 (369bytes) at 1732040931127Finishing writing output for hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-1 so closing down at 1732040931148 (+21 ms)Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/0000000000000000001-wal-1.temp at 1732040931153 (+5 ms)3 split writer threads finished at 1732040931154 (+1 ms)Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732040931168 (+14 ms)Rename recovered edits hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/0000000000000000002 at 1732040931170 (+2 ms)Processed 2 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1732040931174 (+4 ms) 2024-11-19T18:28:51,192 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-2, size=231 (231bytes) 2024-11-19T18:28:51,192 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-2 2024-11-19T18:28:51,194 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-2 after 1ms 2024-11-19T18:28:51,198 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-2 took 6ms 2024-11-19T18:28:51,202 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-2 so closing down 2024-11-19T18:28:51,202 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T18:28:51,205 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-19T18:28:51,206 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/0000000000000000002-wal-2.temp 2024-11-19T18:28:51,206 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T18:28:51,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741842_1018 (size=231) 2024-11-19T18:28:51,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741842_1018 (size=231) 2024-11-19T18:28:51,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741842_1018 (size=231) 2024-11-19T18:28:51,217 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-19T18:28:51,224 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-11-19T18:28:51,226 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-19T18:28:51,226 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-2, journal: Splitting hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-2, size=231 (231bytes) at 1732040931192Finishing writing output for hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-2 so closing down at 1732040931202 (+10 ms)Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/0000000000000000002-wal-2.temp at 1732040931206 (+4 ms)3 split writer threads finished at 1732040931206Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732040931217 (+11 ms)Processed 1 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1732040931226 (+9 ms) 2024-11-19T18:28:51,227 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:28:51,230 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:28:51,251 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal.1732040931231, exclude list is [], retry=0 2024-11-19T18:28:51,257 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:51,258 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:51,259 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:51,263 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal.1732040931231 2024-11-19T18:28:51,264 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:33801:33801)] 2024-11-19T18:28:51,264 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d68d927fde8ce9f3e49247d9ae93404f, NAME => 'testReplayEditsWrittenIntoWAL,,1732040930967.d68d927fde8ce9f3e49247d9ae93404f.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:28:51,264 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732040930967.d68d927fde8ce9f3e49247d9ae93404f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:28:51,264 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d68d927fde8ce9f3e49247d9ae93404f 2024-11-19T18:28:51,264 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d68d927fde8ce9f3e49247d9ae93404f 2024-11-19T18:28:51,267 INFO [StoreOpener-d68d927fde8ce9f3e49247d9ae93404f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d68d927fde8ce9f3e49247d9ae93404f 2024-11-19T18:28:51,268 INFO [StoreOpener-d68d927fde8ce9f3e49247d9ae93404f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d68d927fde8ce9f3e49247d9ae93404f columnFamilyName a 2024-11-19T18:28:51,269 DEBUG [StoreOpener-d68d927fde8ce9f3e49247d9ae93404f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:51,270 INFO [StoreOpener-d68d927fde8ce9f3e49247d9ae93404f-1 {}] regionserver.HStore(327): Store=d68d927fde8ce9f3e49247d9ae93404f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:51,270 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d68d927fde8ce9f3e49247d9ae93404f 2024-11-19T18:28:51,271 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f 2024-11-19T18:28:51,274 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f 2024-11-19T18:28:51,275 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/0000000000000000002 2024-11-19T18:28:51,285 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/0000000000000000002 2024-11-19T18:28:51,289 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d68d927fde8ce9f3e49247d9ae93404f 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-19T18:28:51,346 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/.tmp/a/c4a0d518eb8c49dc92313f3c3d2c12bf is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732040931062/Put/seqid=0 2024-11-19T18:28:51,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741844_1020 (size=5170) 2024-11-19T18:28:51,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741844_1020 (size=5170) 2024-11-19T18:28:51,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741844_1020 (size=5170) 2024-11-19T18:28:51,387 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/.tmp/a/c4a0d518eb8c49dc92313f3c3d2c12bf 2024-11-19T18:28:51,442 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/.tmp/a/c4a0d518eb8c49dc92313f3c3d2c12bf as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/a/c4a0d518eb8c49dc92313f3c3d2c12bf 2024-11-19T18:28:51,453 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/a/c4a0d518eb8c49dc92313f3c3d2c12bf, entries=2, sequenceid=2, filesize=5.0 K 2024-11-19T18:28:51,461 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for d68d927fde8ce9f3e49247d9ae93404f in 169ms, sequenceid=2, compaction requested=false; wal=null 2024-11-19T18:28:51,463 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/0000000000000000002 2024-11-19T18:28:51,464 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d68d927fde8ce9f3e49247d9ae93404f 2024-11-19T18:28:51,464 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d68d927fde8ce9f3e49247d9ae93404f 2024-11-19T18:28:51,467 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d68d927fde8ce9f3e49247d9ae93404f 2024-11-19T18:28:51,471 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/d68d927fde8ce9f3e49247d9ae93404f/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-19T18:28:51,472 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d68d927fde8ce9f3e49247d9ae93404f; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62037312, jitterRate=-0.07557201385498047}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T18:28:51,473 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d68d927fde8ce9f3e49247d9ae93404f: Writing region info on filesystem at 1732040931265Initializing all the Stores at 1732040931266 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040931266Obtaining lock to block concurrent updates at 1732040931289 (+23 ms)Preparing flush snapshotting stores in d68d927fde8ce9f3e49247d9ae93404f at 1732040931289Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732040930967.d68d927fde8ce9f3e49247d9ae93404f., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732040931293 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732040930967.d68d927fde8ce9f3e49247d9ae93404f. at 1732040931293Flushing d68d927fde8ce9f3e49247d9ae93404f/a: creating writer at 1732040931294 (+1 ms)Flushing d68d927fde8ce9f3e49247d9ae93404f/a: appending metadata at 1732040931336 (+42 ms)Flushing d68d927fde8ce9f3e49247d9ae93404f/a: closing flushed file at 1732040931339 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78d2bbf: reopening flushed file at 1732040931440 (+101 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for d68d927fde8ce9f3e49247d9ae93404f in 169ms, sequenceid=2, compaction requested=false; wal=null at 1732040931461 (+21 ms)Cleaning up temporary data from old regions at 1732040931464 (+3 ms)Region opened successfully at 1732040931473 (+9 ms) 2024-11-19T18:28:51,502 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=375 (was 364) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1034186995-172.17.0.2-1732040924630:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:43672 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:56336 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1034186995-172.17.0.2-1732040924630:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:46666 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:52676 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:35734 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:42267/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1034186995-172.17.0.2-1732040924630:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:33336 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 613) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=7477 (was 7547) 2024-11-19T18:28:51,514 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=375, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=7477 2024-11-19T18:28:51,532 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:28:51,535 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:28:51,536 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T18:28:51,540 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-38244650, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/hregion-38244650, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:28:51,554 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-38244650/hregion-38244650.1732040931541, exclude list is [], retry=0 2024-11-19T18:28:51,559 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:51,559 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:51,560 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:51,563 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-38244650/hregion-38244650.1732040931541 2024-11-19T18:28:51,563 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:33801:33801)] 2024-11-19T18:28:51,564 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1bb0d87913124a0042aeaa14d344deda, NAME => 'testReplayEditsWrittenIntoWAL,,1732040931533.1bb0d87913124a0042aeaa14d344deda.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42267/hbase 2024-11-19T18:28:51,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741846_1022 (size=64) 2024-11-19T18:28:51,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741846_1022 (size=64) 2024-11-19T18:28:51,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741846_1022 (size=64) 2024-11-19T18:28:51,576 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732040931533.1bb0d87913124a0042aeaa14d344deda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:28:51,577 INFO [StoreOpener-1bb0d87913124a0042aeaa14d344deda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1bb0d87913124a0042aeaa14d344deda 2024-11-19T18:28:51,580 INFO [StoreOpener-1bb0d87913124a0042aeaa14d344deda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1bb0d87913124a0042aeaa14d344deda columnFamilyName a 2024-11-19T18:28:51,580 DEBUG [StoreOpener-1bb0d87913124a0042aeaa14d344deda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:51,581 INFO [StoreOpener-1bb0d87913124a0042aeaa14d344deda-1 {}] regionserver.HStore(327): Store=1bb0d87913124a0042aeaa14d344deda/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:51,581 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1bb0d87913124a0042aeaa14d344deda 2024-11-19T18:28:51,582 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda 2024-11-19T18:28:51,582 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda 2024-11-19T18:28:51,583 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1bb0d87913124a0042aeaa14d344deda 2024-11-19T18:28:51,583 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1bb0d87913124a0042aeaa14d344deda 2024-11-19T18:28:51,585 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1bb0d87913124a0042aeaa14d344deda 2024-11-19T18:28:51,588 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T18:28:51,589 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1bb0d87913124a0042aeaa14d344deda; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67756723, jitterRate=0.009653851389884949}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T18:28:51,589 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1bb0d87913124a0042aeaa14d344deda: Writing region info on filesystem at 1732040931576Initializing all the Stores at 1732040931577 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040931577Cleaning up temporary data from old regions at 1732040931583 (+6 ms)Region opened successfully at 1732040931589 (+6 ms) 2024-11-19T18:28:51,589 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1bb0d87913124a0042aeaa14d344deda, disabling compactions & flushes 2024-11-19T18:28:51,590 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732040931533.1bb0d87913124a0042aeaa14d344deda. 2024-11-19T18:28:51,590 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732040931533.1bb0d87913124a0042aeaa14d344deda. 2024-11-19T18:28:51,590 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732040931533.1bb0d87913124a0042aeaa14d344deda. after waiting 0 ms 2024-11-19T18:28:51,590 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732040931533.1bb0d87913124a0042aeaa14d344deda. 2024-11-19T18:28:51,590 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732040931533.1bb0d87913124a0042aeaa14d344deda. 2024-11-19T18:28:51,590 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1bb0d87913124a0042aeaa14d344deda: Waiting for close lock at 1732040931589Disabling compacts and flushes for region at 1732040931589Disabling writes for close at 1732040931590 (+1 ms)Writing region close event to WAL at 1732040931590Closed at 1732040931590 2024-11-19T18:28:51,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741845_1021 (size=93) 2024-11-19T18:28:51,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741845_1021 (size=93) 2024-11-19T18:28:51,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741845_1021 (size=93) 2024-11-19T18:28:51,597 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-19T18:28:51,597 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-38244650:(num 1732040931541) 2024-11-19T18:28:51,598 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-19T18:28:51,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741847_1023 (size=369) 2024-11-19T18:28:51,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741847_1023 (size=369) 2024-11-19T18:28:51,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741847_1023 (size=369) 2024-11-19T18:28:51,610 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-19T18:28:51,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741848_1024 (size=231) 2024-11-19T18:28:51,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741848_1024 (size=231) 2024-11-19T18:28:51,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741848_1024 (size=231) 2024-11-19T18:28:51,638 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-2, size=231 (231bytes) 2024-11-19T18:28:51,638 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-2 2024-11-19T18:28:51,639 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-2 after 1ms 2024-11-19T18:28:51,643 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-2 took 5ms 2024-11-19T18:28:51,645 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-2 so closing down 2024-11-19T18:28:51,645 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T18:28:51,648 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-19T18:28:51,649 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000002-wal-2.temp 2024-11-19T18:28:51,650 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T18:28:51,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741849_1025 (size=231) 2024-11-19T18:28:51,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741849_1025 (size=231) 2024-11-19T18:28:51,659 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-19T18:28:51,661 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000002 2024-11-19T18:28:51,662 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-19T18:28:51,662 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-2, journal: Splitting hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-2, size=231 (231bytes) at 1732040931638Finishing writing output for hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-2 so closing down at 1732040931645 (+7 ms)Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000002-wal-2.temp at 1732040931649 (+4 ms)3 split writer threads finished at 1732040931650 (+1 ms)Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732040931659 (+9 ms)Rename recovered edits hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000002 at 1732040931662 (+3 ms)Processed 1 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1732040931662 2024-11-19T18:28:51,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741849_1025 (size=231) 2024-11-19T18:28:51,678 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-1, size=369 (369bytes) 2024-11-19T18:28:51,678 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-1 2024-11-19T18:28:51,678 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-1 after 0ms 2024-11-19T18:28:51,682 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-1 took 5ms 2024-11-19T18:28:51,684 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-1 so closing down 2024-11-19T18:28:51,684 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T18:28:51,686 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-19T18:28:51,688 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000001-wal-1.temp 2024-11-19T18:28:51,688 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T18:28:51,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741850_1026 (size=369) 2024-11-19T18:28:51,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741850_1026 (size=369) 2024-11-19T18:28:51,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741850_1026 (size=369) 2024-11-19T18:28:51,698 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-19T18:28:51,705 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000002, length=231 2024-11-19T18:28:51,707 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000002 2024-11-19T18:28:51,708 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-19T18:28:51,708 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-1, journal: Splitting hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-1, size=369 (369bytes) at 1732040931678Finishing writing output for hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-1 so closing down at 1732040931684 (+6 ms)Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000001-wal-1.temp at 1732040931688 (+4 ms)3 split writer threads finished at 1732040931688Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732040931698 (+10 ms)Rename recovered edits hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000002 at 1732040931707 (+9 ms)Processed 2 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1732040931708 (+1 ms) 2024-11-19T18:28:51,708 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:28:51,711 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:28:51,726 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal.1732040931712, exclude list is [], retry=0 2024-11-19T18:28:51,730 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:51,731 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:51,731 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:51,734 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal.1732040931712 2024-11-19T18:28:51,734 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:37769:37769)] 2024-11-19T18:28:51,734 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1bb0d87913124a0042aeaa14d344deda, NAME => 'testReplayEditsWrittenIntoWAL,,1732040931533.1bb0d87913124a0042aeaa14d344deda.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:28:51,735 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732040931533.1bb0d87913124a0042aeaa14d344deda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:28:51,735 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1bb0d87913124a0042aeaa14d344deda 2024-11-19T18:28:51,735 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1bb0d87913124a0042aeaa14d344deda 2024-11-19T18:28:51,737 INFO [StoreOpener-1bb0d87913124a0042aeaa14d344deda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1bb0d87913124a0042aeaa14d344deda 2024-11-19T18:28:51,739 INFO [StoreOpener-1bb0d87913124a0042aeaa14d344deda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1bb0d87913124a0042aeaa14d344deda columnFamilyName a 2024-11-19T18:28:51,739 DEBUG [StoreOpener-1bb0d87913124a0042aeaa14d344deda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:51,740 INFO [StoreOpener-1bb0d87913124a0042aeaa14d344deda-1 {}] regionserver.HStore(327): Store=1bb0d87913124a0042aeaa14d344deda/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:51,740 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1bb0d87913124a0042aeaa14d344deda 2024-11-19T18:28:51,741 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda 2024-11-19T18:28:51,744 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda 2024-11-19T18:28:51,745 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000002 2024-11-19T18:28:51,750 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000002 2024-11-19T18:28:51,750 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1bb0d87913124a0042aeaa14d344deda 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-19T18:28:51,767 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/.tmp/a/5c4cfe45cee146bba5d2ebd320c22039 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732040931597/Put/seqid=0 2024-11-19T18:28:51,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741852_1028 (size=5170) 2024-11-19T18:28:51,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741852_1028 (size=5170) 2024-11-19T18:28:51,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741852_1028 (size=5170) 2024-11-19T18:28:51,778 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/.tmp/a/5c4cfe45cee146bba5d2ebd320c22039 2024-11-19T18:28:51,786 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/.tmp/a/5c4cfe45cee146bba5d2ebd320c22039 as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/a/5c4cfe45cee146bba5d2ebd320c22039 2024-11-19T18:28:51,795 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/a/5c4cfe45cee146bba5d2ebd320c22039, entries=2, sequenceid=2, filesize=5.0 K 2024-11-19T18:28:51,795 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1bb0d87913124a0042aeaa14d344deda in 45ms, sequenceid=2, compaction requested=false; wal=null 2024-11-19T18:28:51,796 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/0000000000000000002 2024-11-19T18:28:51,797 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1bb0d87913124a0042aeaa14d344deda 2024-11-19T18:28:51,797 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1bb0d87913124a0042aeaa14d344deda 2024-11-19T18:28:51,800 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1bb0d87913124a0042aeaa14d344deda 2024-11-19T18:28:51,803 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/1bb0d87913124a0042aeaa14d344deda/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-19T18:28:51,804 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1bb0d87913124a0042aeaa14d344deda; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67257358, jitterRate=0.002212733030319214}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T18:28:51,804 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1bb0d87913124a0042aeaa14d344deda: Writing region info on filesystem at 1732040931735Initializing all the Stores at 1732040931737 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040931737Obtaining lock to block concurrent updates at 1732040931750 (+13 ms)Preparing flush snapshotting stores in 1bb0d87913124a0042aeaa14d344deda at 1732040931750Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732040931533.1bb0d87913124a0042aeaa14d344deda., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732040931750Flushing stores of testReplayEditsWrittenIntoWAL,,1732040931533.1bb0d87913124a0042aeaa14d344deda. at 1732040931750Flushing 1bb0d87913124a0042aeaa14d344deda/a: creating writer at 1732040931750Flushing 1bb0d87913124a0042aeaa14d344deda/a: appending metadata at 1732040931766 (+16 ms)Flushing 1bb0d87913124a0042aeaa14d344deda/a: closing flushed file at 1732040931766Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13c44559: reopening flushed file at 1732040931785 (+19 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1bb0d87913124a0042aeaa14d344deda in 45ms, sequenceid=2, compaction requested=false; wal=null at 1732040931795 (+10 ms)Cleaning up temporary data from old regions at 1732040931797 (+2 ms)Region opened successfully at 1732040931804 (+7 ms) 2024-11-19T18:28:51,823 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=385 (was 375) Potentially hanging thread: PacketResponder: BP-1034186995-172.17.0.2-1732040924630:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:56336 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:33406 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:43672 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:52676 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1034186995-172.17.0.2-1732040924630:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:46732 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1034186995-172.17.0.2-1732040924630:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:35800 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=165 (was 127) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7471 (was 7477) 2024-11-19T18:28:51,835 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=385, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=165, ProcessCount=11, AvailableMemoryMB=7471 2024-11-19T18:28:51,851 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:28:51,854 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:28:51,855 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T18:28:51,859 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-70231732, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/hregion-70231732, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:28:51,876 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-70231732/hregion-70231732.1732040931860, exclude list is [], retry=0 2024-11-19T18:28:51,880 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:51,881 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:51,881 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:51,885 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-70231732/hregion-70231732.1732040931860 2024-11-19T18:28:51,886 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:37769:37769)] 2024-11-19T18:28:51,886 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => dc70bfaaa956b1bc5c140ae3528ed710, NAME => 'testReplayEditsWrittenIntoWAL,,1732040931852.dc70bfaaa956b1bc5c140ae3528ed710.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42267/hbase 2024-11-19T18:28:51,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741854_1030 (size=64) 2024-11-19T18:28:51,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741854_1030 (size=64) 2024-11-19T18:28:51,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741854_1030 (size=64) 2024-11-19T18:28:51,899 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732040931852.dc70bfaaa956b1bc5c140ae3528ed710.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:28:51,901 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:51,902 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc70bfaaa956b1bc5c140ae3528ed710 columnFamilyName a 2024-11-19T18:28:51,903 DEBUG [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:51,903 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] regionserver.HStore(327): Store=dc70bfaaa956b1bc5c140ae3528ed710/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:51,903 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:51,905 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc70bfaaa956b1bc5c140ae3528ed710 columnFamilyName b 2024-11-19T18:28:51,905 DEBUG [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:51,906 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] regionserver.HStore(327): Store=dc70bfaaa956b1bc5c140ae3528ed710/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:51,906 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:51,908 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc70bfaaa956b1bc5c140ae3528ed710 columnFamilyName c 2024-11-19T18:28:51,908 DEBUG [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:51,909 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] regionserver.HStore(327): Store=dc70bfaaa956b1bc5c140ae3528ed710/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:51,909 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:51,910 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:51,910 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:51,911 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:51,912 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:51,912 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:28:51,914 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:51,916 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T18:28:51,917 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dc70bfaaa956b1bc5c140ae3528ed710; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74557599, jitterRate=0.11099480092525482}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:28:51,917 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dc70bfaaa956b1bc5c140ae3528ed710: Writing region info on filesystem at 1732040931899Initializing all the Stores at 1732040931900 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040931900Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040931900Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040931900Cleaning up temporary data from old regions at 1732040931912 (+12 ms)Region opened successfully at 1732040931917 (+5 ms) 2024-11-19T18:28:51,917 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dc70bfaaa956b1bc5c140ae3528ed710, disabling compactions & flushes 2024-11-19T18:28:51,917 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732040931852.dc70bfaaa956b1bc5c140ae3528ed710. 2024-11-19T18:28:51,917 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732040931852.dc70bfaaa956b1bc5c140ae3528ed710. 2024-11-19T18:28:51,917 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732040931852.dc70bfaaa956b1bc5c140ae3528ed710. after waiting 0 ms 2024-11-19T18:28:51,918 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732040931852.dc70bfaaa956b1bc5c140ae3528ed710. 2024-11-19T18:28:51,918 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732040931852.dc70bfaaa956b1bc5c140ae3528ed710. 2024-11-19T18:28:51,918 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dc70bfaaa956b1bc5c140ae3528ed710: Waiting for close lock at 1732040931917Disabling compacts and flushes for region at 1732040931917Disabling writes for close at 1732040931917Writing region close event to WAL at 1732040931918 (+1 ms)Closed at 1732040931918 2024-11-19T18:28:51,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741853_1029 (size=93) 2024-11-19T18:28:51,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741853_1029 (size=93) 2024-11-19T18:28:51,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741853_1029 (size=93) 2024-11-19T18:28:51,925 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-19T18:28:51,925 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-70231732:(num 1732040931860) 2024-11-19T18:28:51,925 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:28:51,927 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:28:51,942 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928, exclude list is [], retry=0 2024-11-19T18:28:51,945 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:51,946 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:51,946 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:51,949 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928 2024-11-19T18:28:51,949 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:37769:37769)] 2024-11-19T18:28:52,130 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928, size=0 (0bytes) 2024-11-19T18:28:52,131 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928 might be still open, length is 0 2024-11-19T18:28:52,131 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928 2024-11-19T18:28:52,132 WARN [IPC Server handler 2 on default port 42267 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-19T18:28:52,132 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928 after 1ms 2024-11-19T18:28:52,626 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:35830 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41941:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35830 dst: /127.0.0.1:41941 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41941 remote=/127.0.0.1:35830]. Total timeout mills is 60000, 59470 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:28:52,628 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:46770 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33471:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46770 dst: /127.0.0.1:33471 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:28:52,628 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:33446 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33841:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33446 dst: /127.0.0.1:33841 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:28:52,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741855_1032 (size=470675) 2024-11-19T18:28:52,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741855_1032 (size=470675) 2024-11-19T18:28:52,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741855_1032 (size=470675) 2024-11-19T18:28:56,033 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T18:28:56,082 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T18:28:56,133 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928 after 4002ms 2024-11-19T18:28:56,139 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928 took 4009ms 2024-11-19T18:28:56,145 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732040931928.temp 2024-11-19T18:28:56,151 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/recovered.edits/0000000000000000001-wal.1732040931928.temp 2024-11-19T18:28:56,222 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928; continuing. 2024-11-19T18:28:56,222 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928 so closing down 2024-11-19T18:28:56,222 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T18:28:56,222 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T18:28:56,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741856_1033 (size=470683) 2024-11-19T18:28:56,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741856_1033 (size=470683) 2024-11-19T18:28:56,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741856_1033 (size=470683) 2024-11-19T18:28:56,227 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/recovered.edits/0000000000000000001-wal.1732040931928.temp (wrote 3002 edits, skipped 0 edits in 30 ms) 2024-11-19T18:28:56,229 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/recovered.edits/0000000000000000001-wal.1732040931928.temp to hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/recovered.edits/0000000000000003002 2024-11-19T18:28:56,229 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 89 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928, size=0, length=0, corrupted=false, cancelled=false 2024-11-19T18:28:56,229 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928, journal: Splitting hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928, size=0 (0bytes) at 1732040932130Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/recovered.edits/0000000000000000001-wal.1732040931928.temp at 1732040936152 (+4022 ms)Split 1024 edits, skipped 0 edits. at 1732040936190 (+38 ms)Split 2048 edits, skipped 0 edits. at 1732040936209 (+19 ms)Finishing writing output for hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928 so closing down at 1732040936222 (+13 ms)3 split writer threads finished at 1732040936222Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/recovered.edits/0000000000000000001-wal.1732040931928.temp (wrote 3002 edits, skipped 0 edits in 30 ms) at 1732040936227 (+5 ms)Rename recovered edits hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/recovered.edits/0000000000000000001-wal.1732040931928.temp to hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/recovered.edits/0000000000000003002 at 1732040936229 (+2 ms)Processed 3002 edits across 1 Regions in 89 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928, size=0, length=0, corrupted=false, cancelled=false at 1732040936229 2024-11-19T18:28:56,232 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928 to hdfs://localhost:42267/hbase/oldWALs/wal.1732040931928 2024-11-19T18:28:56,234 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/recovered.edits/0000000000000003002 2024-11-19T18:28:56,234 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:28:56,237 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:28:56,251 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040936237, exclude list is [], retry=0 2024-11-19T18:28:56,255 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:56,255 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:56,256 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:56,258 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040936237 2024-11-19T18:28:56,258 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:40377:40377)] 2024-11-19T18:28:56,259 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732040931852.dc70bfaaa956b1bc5c140ae3528ed710.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:28:56,260 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:56,261 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc70bfaaa956b1bc5c140ae3528ed710 columnFamilyName a 2024-11-19T18:28:56,261 DEBUG [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:56,262 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] regionserver.HStore(327): Store=dc70bfaaa956b1bc5c140ae3528ed710/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:56,262 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:56,263 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc70bfaaa956b1bc5c140ae3528ed710 columnFamilyName b 2024-11-19T18:28:56,263 DEBUG [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:56,263 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] regionserver.HStore(327): Store=dc70bfaaa956b1bc5c140ae3528ed710/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:56,264 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:56,264 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc70bfaaa956b1bc5c140ae3528ed710 columnFamilyName c 2024-11-19T18:28:56,265 DEBUG [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:56,265 INFO [StoreOpener-dc70bfaaa956b1bc5c140ae3528ed710-1 {}] regionserver.HStore(327): Store=dc70bfaaa956b1bc5c140ae3528ed710/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:56,265 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:56,266 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:56,268 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:56,269 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/recovered.edits/0000000000000003002 2024-11-19T18:28:56,300 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-19T18:28:56,680 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dc70bfaaa956b1bc5c140ae3528ed710 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-19T18:28:56,718 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/a/bca4c36bec1d4c16bb66492420389058 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732040931955/Put/seqid=0 2024-11-19T18:28:56,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741858_1035 (size=50463) 2024-11-19T18:28:56,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741858_1035 (size=50463) 2024-11-19T18:28:56,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741858_1035 (size=50463) 2024-11-19T18:28:56,728 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/a/bca4c36bec1d4c16bb66492420389058 2024-11-19T18:28:56,736 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/a/bca4c36bec1d4c16bb66492420389058 as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/a/bca4c36bec1d4c16bb66492420389058 2024-11-19T18:28:56,743 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/a/bca4c36bec1d4c16bb66492420389058, entries=754, sequenceid=754, filesize=49.3 K 2024-11-19T18:28:56,743 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for dc70bfaaa956b1bc5c140ae3528ed710 in 63ms, sequenceid=754, compaction requested=false; wal=null 2024-11-19T18:28:56,759 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-19T18:28:56,759 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dc70bfaaa956b1bc5c140ae3528ed710 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-19T18:28:56,769 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/a/bf5d9ef5f5594d9e999a0c922f629a0e is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732040931992/Put/seqid=0 2024-11-19T18:28:56,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741859_1036 (size=20072) 2024-11-19T18:28:56,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741859_1036 (size=20072) 2024-11-19T18:28:56,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741859_1036 (size=20072) 2024-11-19T18:28:56,779 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/a/bf5d9ef5f5594d9e999a0c922f629a0e 2024-11-19T18:28:56,811 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/b/45e6ec492cec4ea98ebe5831b401c170 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732040932010/Put/seqid=0 2024-11-19T18:28:56,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741860_1037 (size=35835) 2024-11-19T18:28:56,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741860_1037 (size=35835) 2024-11-19T18:28:56,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741860_1037 (size=35835) 2024-11-19T18:28:56,821 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/b/45e6ec492cec4ea98ebe5831b401c170 2024-11-19T18:28:56,828 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/a/bf5d9ef5f5594d9e999a0c922f629a0e as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/a/bf5d9ef5f5594d9e999a0c922f629a0e 2024-11-19T18:28:56,837 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/a/bf5d9ef5f5594d9e999a0c922f629a0e, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-19T18:28:56,839 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/b/45e6ec492cec4ea98ebe5831b401c170 as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/b/45e6ec492cec4ea98ebe5831b401c170 2024-11-19T18:28:56,846 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/b/45e6ec492cec4ea98ebe5831b401c170, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-19T18:28:56,846 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for dc70bfaaa956b1bc5c140ae3528ed710 in 87ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-19T18:28:56,856 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-19T18:28:56,857 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dc70bfaaa956b1bc5c140ae3528ed710 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-19T18:28:56,864 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/b/73146da961b04158a56fb3678efbccdc is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732040932029/Put/seqid=0 2024-11-19T18:28:56,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741861_1038 (size=35082) 2024-11-19T18:28:56,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741861_1038 (size=35082) 2024-11-19T18:28:56,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741861_1038 (size=35082) 2024-11-19T18:28:56,875 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/b/73146da961b04158a56fb3678efbccdc 2024-11-19T18:28:56,902 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/c/ae87feeccd474c46a8523028e81ec4c7 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732040932055/Put/seqid=0 2024-11-19T18:28:56,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741862_1039 (size=20825) 2024-11-19T18:28:56,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741862_1039 (size=20825) 2024-11-19T18:28:56,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741862_1039 (size=20825) 2024-11-19T18:28:56,911 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/c/ae87feeccd474c46a8523028e81ec4c7 2024-11-19T18:28:56,919 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/b/73146da961b04158a56fb3678efbccdc as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/b/73146da961b04158a56fb3678efbccdc 2024-11-19T18:28:56,925 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/b/73146da961b04158a56fb3678efbccdc, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-19T18:28:56,927 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/c/ae87feeccd474c46a8523028e81ec4c7 as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/c/ae87feeccd474c46a8523028e81ec4c7 2024-11-19T18:28:56,934 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/c/ae87feeccd474c46a8523028e81ec4c7, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-19T18:28:56,935 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for dc70bfaaa956b1bc5c140ae3528ed710 in 77ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-19T18:28:56,944 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732040932094/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732040931852.dc70bfaaa956b1bc5c140ae3528ed710. 2024-11-19T18:28:56,949 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/recovered.edits/0000000000000003002 2024-11-19T18:28:56,949 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-19T18:28:56,950 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dc70bfaaa956b1bc5c140ae3528ed710 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-19T18:28:56,960 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/c/3983fcadb42b4975899b6328d12befe4 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732040932062/Put/seqid=0 2024-11-19T18:28:56,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741863_1040 (size=50301) 2024-11-19T18:28:56,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741863_1040 (size=50301) 2024-11-19T18:28:56,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741863_1040 (size=50301) 2024-11-19T18:28:56,971 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/c/3983fcadb42b4975899b6328d12befe4 2024-11-19T18:28:56,978 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3983fcadb42b4975899b6328d12befe4 2024-11-19T18:28:56,979 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/.tmp/c/3983fcadb42b4975899b6328d12befe4 as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/c/3983fcadb42b4975899b6328d12befe4 2024-11-19T18:28:56,985 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3983fcadb42b4975899b6328d12befe4 2024-11-19T18:28:56,985 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/c/3983fcadb42b4975899b6328d12befe4, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-19T18:28:56,985 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for dc70bfaaa956b1bc5c140ae3528ed710 in 36ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-19T18:28:56,986 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/recovered.edits/0000000000000003002 2024-11-19T18:28:56,987 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:56,987 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:56,988 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T18:28:56,990 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dc70bfaaa956b1bc5c140ae3528ed710 2024-11-19T18:28:56,992 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenIntoWAL/dc70bfaaa956b1bc5c140ae3528ed710/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-19T18:28:56,993 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dc70bfaaa956b1bc5c140ae3528ed710; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59985049, jitterRate=-0.10615311563014984}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T18:28:56,993 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dc70bfaaa956b1bc5c140ae3528ed710: Writing region info on filesystem at 1732040936259Initializing all the Stores at 1732040936260 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040936260Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040936260Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040936260Cleaning up temporary data from old regions at 1732040936987 (+727 ms)Region opened successfully at 1732040936993 (+6 ms) 2024-11-19T18:28:57,060 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dc70bfaaa956b1bc5c140ae3528ed710, disabling compactions & flushes 2024-11-19T18:28:57,060 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732040931852.dc70bfaaa956b1bc5c140ae3528ed710. 2024-11-19T18:28:57,060 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732040931852.dc70bfaaa956b1bc5c140ae3528ed710. 2024-11-19T18:28:57,060 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732040931852.dc70bfaaa956b1bc5c140ae3528ed710. after waiting 0 ms 2024-11-19T18:28:57,060 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732040931852.dc70bfaaa956b1bc5c140ae3528ed710. 2024-11-19T18:28:57,062 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732040931852.dc70bfaaa956b1bc5c140ae3528ed710. 2024-11-19T18:28:57,062 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dc70bfaaa956b1bc5c140ae3528ed710: Waiting for close lock at 1732040937060Disabling compacts and flushes for region at 1732040937060Disabling writes for close at 1732040937060Writing region close event to WAL at 1732040937062 (+2 ms)Closed at 1732040937062 2024-11-19T18:28:57,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741857_1034 (size=93) 2024-11-19T18:28:57,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741857_1034 (size=93) 2024-11-19T18:28:57,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741857_1034 (size=93) 2024-11-19T18:28:57,069 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-19T18:28:57,069 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732040936237) 2024-11-19T18:28:57,089 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=401 (was 385) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_368631960_22 at /127.0.0.1:35848 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_368631960_22 at /127.0.0.1:33478 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_368631960_22 at /127.0.0.1:33514 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:46019 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_368631960_22 at /127.0.0.1:46826 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:42267 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46019 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46199 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:46199 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:42267 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=184 (was 165) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7420 (was 7471) 2024-11-19T18:28:57,105 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=401, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=184, ProcessCount=11, AvailableMemoryMB=7417 2024-11-19T18:28:57,126 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:28:57,128 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:28:57,129 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T18:28:57,132 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-33219676, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/hregion-33219676, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:28:57,146 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-33219676/hregion-33219676.1732040937132, exclude list is [], retry=0 2024-11-19T18:28:57,150 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:57,151 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:57,151 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:57,153 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-33219676/hregion-33219676.1732040937132 2024-11-19T18:28:57,154 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:37769:37769)] 2024-11-19T18:28:57,154 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9ae74df7e71a237c9c1e150fa4bda3ef, NAME => 'test2727,,1732040937126.9ae74df7e71a237c9c1e150fa4bda3ef.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42267/hbase 2024-11-19T18:28:57,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741865_1042 (size=43) 2024-11-19T18:28:57,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741865_1042 (size=43) 2024-11-19T18:28:57,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741865_1042 (size=43) 2024-11-19T18:28:57,167 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732040937126.9ae74df7e71a237c9c1e150fa4bda3ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:28:57,168 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,170 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ae74df7e71a237c9c1e150fa4bda3ef columnFamilyName a 2024-11-19T18:28:57,170 DEBUG [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:57,171 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] regionserver.HStore(327): Store=9ae74df7e71a237c9c1e150fa4bda3ef/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:57,171 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,172 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ae74df7e71a237c9c1e150fa4bda3ef columnFamilyName b 2024-11-19T18:28:57,172 DEBUG [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:57,173 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] regionserver.HStore(327): Store=9ae74df7e71a237c9c1e150fa4bda3ef/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:57,173 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,174 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ae74df7e71a237c9c1e150fa4bda3ef columnFamilyName c 2024-11-19T18:28:57,174 DEBUG [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:57,175 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] regionserver.HStore(327): Store=9ae74df7e71a237c9c1e150fa4bda3ef/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:57,175 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,176 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,176 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,178 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,178 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,179 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:28:57,180 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,183 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T18:28:57,184 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9ae74df7e71a237c9c1e150fa4bda3ef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68912419, jitterRate=0.02687506377696991}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:28:57,185 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9ae74df7e71a237c9c1e150fa4bda3ef: Writing region info on filesystem at 1732040937167Initializing all the Stores at 1732040937168 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040937168Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040937168Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040937168Cleaning up temporary data from old regions at 1732040937178 (+10 ms)Region opened successfully at 1732040937185 (+7 ms) 2024-11-19T18:28:57,185 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9ae74df7e71a237c9c1e150fa4bda3ef, disabling compactions & flushes 2024-11-19T18:28:57,186 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732040937126.9ae74df7e71a237c9c1e150fa4bda3ef. 2024-11-19T18:28:57,186 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732040937126.9ae74df7e71a237c9c1e150fa4bda3ef. 2024-11-19T18:28:57,186 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732040937126.9ae74df7e71a237c9c1e150fa4bda3ef. after waiting 0 ms 2024-11-19T18:28:57,186 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732040937126.9ae74df7e71a237c9c1e150fa4bda3ef. 2024-11-19T18:28:57,186 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732040937126.9ae74df7e71a237c9c1e150fa4bda3ef. 2024-11-19T18:28:57,186 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9ae74df7e71a237c9c1e150fa4bda3ef: Waiting for close lock at 1732040937185Disabling compacts and flushes for region at 1732040937185Disabling writes for close at 1732040937186 (+1 ms)Writing region close event to WAL at 1732040937186Closed at 1732040937186 2024-11-19T18:28:57,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741864_1041 (size=93) 2024-11-19T18:28:57,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741864_1041 (size=93) 2024-11-19T18:28:57,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741864_1041 (size=93) 2024-11-19T18:28:57,193 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-19T18:28:57,193 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-33219676:(num 1732040937132) 2024-11-19T18:28:57,193 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:28:57,196 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:28:57,217 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937196, exclude list is [], retry=0 2024-11-19T18:28:57,221 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:57,221 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:57,222 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:57,224 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937196 2024-11-19T18:28:57,225 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:33801:33801)] 2024-11-19T18:28:57,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741866_1043 (size=344306) 2024-11-19T18:28:57,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741866_1043 (size=344306) 2024-11-19T18:28:57,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741866_1043 (size=344306) 2024-11-19T18:28:57,410 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937196, size=336.2 K (344306bytes) 2024-11-19T18:28:57,410 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937196 2024-11-19T18:28:57,411 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937196 after 1ms 2024-11-19T18:28:57,416 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937196 took 5ms 2024-11-19T18:28:57,421 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732040937196.temp 2024-11-19T18:28:57,422 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000000001-wal.1732040937196.temp 2024-11-19T18:28:57,442 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937196 so closing down 2024-11-19T18:28:57,442 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T18:28:57,442 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T18:28:57,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741867_1044 (size=344306) 2024-11-19T18:28:57,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741867_1044 (size=344306) 2024-11-19T18:28:57,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741867_1044 (size=344306) 2024-11-19T18:28:57,447 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000000001-wal.1732040937196.temp (wrote 3000 edits, skipped 0 edits in 11 ms) 2024-11-19T18:28:57,449 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000000001-wal.1732040937196.temp to hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000003000 2024-11-19T18:28:57,450 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937196, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-11-19T18:28:57,450 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937196, journal: Splitting hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937196, size=336.2 K (344306bytes) at 1732040937410Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000000001-wal.1732040937196.temp at 1732040937422 (+12 ms)Split 1024 edits, skipped 0 edits. at 1732040937425 (+3 ms)Split 2048 edits, skipped 0 edits. at 1732040937433 (+8 ms)Finishing writing output for hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937196 so closing down at 1732040937442 (+9 ms)3 split writer threads finished at 1732040937442Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000000001-wal.1732040937196.temp (wrote 3000 edits, skipped 0 edits in 11 ms) at 1732040937447 (+5 ms)Rename recovered edits hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000000001-wal.1732040937196.temp to hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000003000 at 1732040937449 (+2 ms)Processed 3000 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937196, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1732040937450 (+1 ms) 2024-11-19T18:28:57,452 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937196 to hdfs://localhost:42267/hbase/oldWALs/wal.1732040937196 2024-11-19T18:28:57,453 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000003000 2024-11-19T18:28:57,453 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:28:57,455 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:28:57,470 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937455, exclude list is [], retry=0 2024-11-19T18:28:57,474 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:57,474 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:57,475 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:57,481 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937455 2024-11-19T18:28:57,481 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:40377:40377)] 2024-11-19T18:28:57,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741868_1045 (size=344433) 2024-11-19T18:28:57,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741868_1045 (size=344433) 2024-11-19T18:28:57,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741868_1045 (size=344433) 2024-11-19T18:28:57,631 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937455, size=336.4 K (344433bytes) 2024-11-19T18:28:57,631 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937455 2024-11-19T18:28:57,632 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937455 after 1ms 2024-11-19T18:28:57,636 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937455 took 4ms 2024-11-19T18:28:57,640 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732040937455.temp 2024-11-19T18:28:57,641 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000003001-wal.1732040937455.temp 2024-11-19T18:28:57,671 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937455 so closing down 2024-11-19T18:28:57,671 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T18:28:57,671 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T18:28:57,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741869_1046 (size=344433) 2024-11-19T18:28:57,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741869_1046 (size=344433) 2024-11-19T18:28:57,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741869_1046 (size=344433) 2024-11-19T18:28:57,676 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000003001-wal.1732040937455.temp (wrote 3000 edits, skipped 0 edits in 11 ms) 2024-11-19T18:28:57,678 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000003001-wal.1732040937455.temp to hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000006000 2024-11-19T18:28:57,678 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937455, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-11-19T18:28:57,678 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937455, journal: Splitting hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937455, size=336.4 K (344433bytes) at 1732040937631Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000003001-wal.1732040937455.temp at 1732040937642 (+11 ms)Split 1024 edits, skipped 0 edits. at 1732040937648 (+6 ms)Split 2048 edits, skipped 0 edits. at 1732040937661 (+13 ms)Finishing writing output for hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937455 so closing down at 1732040937671 (+10 ms)3 split writer threads finished at 1732040937671Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000003001-wal.1732040937455.temp (wrote 3000 edits, skipped 0 edits in 11 ms) at 1732040937676 (+5 ms)Rename recovered edits hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000003001-wal.1732040937455.temp to hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000006000 at 1732040937678 (+2 ms)Processed 3000 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937455, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1732040937678 2024-11-19T18:28:57,680 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937455 to hdfs://localhost:42267/hbase/oldWALs/wal.1732040937455 2024-11-19T18:28:57,681 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000006000 2024-11-19T18:28:57,681 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:28:57,683 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/test2727-manual,16010,1732040937125, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:28:57,698 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937684, exclude list is [], retry=0 2024-11-19T18:28:57,702 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:57,702 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:57,702 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:57,705 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732040937125/wal.1732040937684 2024-11-19T18:28:57,706 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:40377:40377)] 2024-11-19T18:28:57,706 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9ae74df7e71a237c9c1e150fa4bda3ef, NAME => 'test2727,,1732040937126.9ae74df7e71a237c9c1e150fa4bda3ef.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:28:57,706 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732040937126.9ae74df7e71a237c9c1e150fa4bda3ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:28:57,706 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,706 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,708 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,710 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ae74df7e71a237c9c1e150fa4bda3ef columnFamilyName a 2024-11-19T18:28:57,710 DEBUG [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:57,710 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] regionserver.HStore(327): Store=9ae74df7e71a237c9c1e150fa4bda3ef/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:57,711 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,712 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ae74df7e71a237c9c1e150fa4bda3ef columnFamilyName b 2024-11-19T18:28:57,712 DEBUG [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:57,713 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] regionserver.HStore(327): Store=9ae74df7e71a237c9c1e150fa4bda3ef/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:57,713 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,714 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ae74df7e71a237c9c1e150fa4bda3ef columnFamilyName c 2024-11-19T18:28:57,714 DEBUG [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:57,714 INFO [StoreOpener-9ae74df7e71a237c9c1e150fa4bda3ef-1 {}] regionserver.HStore(327): Store=9ae74df7e71a237c9c1e150fa4bda3ef/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:57,715 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,716 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,718 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,719 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000003000 2024-11-19T18:28:57,754 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000003000 2024-11-19T18:28:57,755 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000006000 2024-11-19T18:28:57,792 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000006000 2024-11-19T18:28:57,792 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9ae74df7e71a237c9c1e150fa4bda3ef 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-19T18:28:57,816 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/.tmp/a/c6b9ccf3f37f4938833851d73754bb4a is 41, key is test2727/a:100/1732040937484/Put/seqid=0 2024-11-19T18:28:57,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741871_1048 (size=84227) 2024-11-19T18:28:57,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741871_1048 (size=84227) 2024-11-19T18:28:57,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741871_1048 (size=84227) 2024-11-19T18:28:57,827 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/.tmp/a/c6b9ccf3f37f4938833851d73754bb4a 2024-11-19T18:28:57,859 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/.tmp/b/cad12a2658404d5a85403cb076e378fd is 41, key is test2727/b:100/1732040937525/Put/seqid=0 2024-11-19T18:28:57,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741872_1049 (size=84609) 2024-11-19T18:28:57,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741872_1049 (size=84609) 2024-11-19T18:28:57,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741872_1049 (size=84609) 2024-11-19T18:28:57,865 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/.tmp/b/cad12a2658404d5a85403cb076e378fd 2024-11-19T18:28:57,903 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/.tmp/c/5dc2571520dc481c8866705569c3f042 is 41, key is test2727/c:100/1732040937572/Put/seqid=0 2024-11-19T18:28:57,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741873_1050 (size=84609) 2024-11-19T18:28:57,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741873_1050 (size=84609) 2024-11-19T18:28:57,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741873_1050 (size=84609) 2024-11-19T18:28:57,910 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/.tmp/c/5dc2571520dc481c8866705569c3f042 2024-11-19T18:28:57,918 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/.tmp/a/c6b9ccf3f37f4938833851d73754bb4a as hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/a/c6b9ccf3f37f4938833851d73754bb4a 2024-11-19T18:28:57,927 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/a/c6b9ccf3f37f4938833851d73754bb4a, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-19T18:28:57,928 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/.tmp/b/cad12a2658404d5a85403cb076e378fd as hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/b/cad12a2658404d5a85403cb076e378fd 2024-11-19T18:28:57,937 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/b/cad12a2658404d5a85403cb076e378fd, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-19T18:28:57,938 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/.tmp/c/5dc2571520dc481c8866705569c3f042 as hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/c/5dc2571520dc481c8866705569c3f042 2024-11-19T18:28:57,947 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/c/5dc2571520dc481c8866705569c3f042, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-19T18:28:57,947 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 9ae74df7e71a237c9c1e150fa4bda3ef in 155ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-19T18:28:57,948 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000003000 2024-11-19T18:28:57,949 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/0000000000000006000 2024-11-19T18:28:57,950 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,950 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,951 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:28:57,953 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9ae74df7e71a237c9c1e150fa4bda3ef 2024-11-19T18:28:57,956 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/test2727/9ae74df7e71a237c9c1e150fa4bda3ef/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-19T18:28:57,957 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9ae74df7e71a237c9c1e150fa4bda3ef; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60436133, jitterRate=-0.09943144023418427}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:28:57,958 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9ae74df7e71a237c9c1e150fa4bda3ef: Writing region info on filesystem at 1732040937707Initializing all the Stores at 1732040937708 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040937708Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040937708Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040937708Obtaining lock to block concurrent updates at 1732040937792 (+84 ms)Preparing flush snapshotting stores in 9ae74df7e71a237c9c1e150fa4bda3ef at 1732040937792Finished memstore snapshotting test2727,,1732040937126.9ae74df7e71a237c9c1e150fa4bda3ef., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732040937792Flushing stores of test2727,,1732040937126.9ae74df7e71a237c9c1e150fa4bda3ef. at 1732040937792Flushing 9ae74df7e71a237c9c1e150fa4bda3ef/a: creating writer at 1732040937792Flushing 9ae74df7e71a237c9c1e150fa4bda3ef/a: appending metadata at 1732040937816 (+24 ms)Flushing 9ae74df7e71a237c9c1e150fa4bda3ef/a: closing flushed file at 1732040937816Flushing 9ae74df7e71a237c9c1e150fa4bda3ef/b: creating writer at 1732040937834 (+18 ms)Flushing 9ae74df7e71a237c9c1e150fa4bda3ef/b: appending metadata at 1732040937856 (+22 ms)Flushing 9ae74df7e71a237c9c1e150fa4bda3ef/b: closing flushed file at 1732040937857 (+1 ms)Flushing 9ae74df7e71a237c9c1e150fa4bda3ef/c: creating writer at 1732040937872 (+15 ms)Flushing 9ae74df7e71a237c9c1e150fa4bda3ef/c: appending metadata at 1732040937902 (+30 ms)Flushing 9ae74df7e71a237c9c1e150fa4bda3ef/c: closing flushed file at 1732040937902Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11c3ae8b: reopening flushed file at 1732040937917 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@568af8da: reopening flushed file at 1732040937927 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51b4bfca: reopening flushed file at 1732040937937 (+10 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 9ae74df7e71a237c9c1e150fa4bda3ef in 155ms, sequenceid=6000, compaction requested=false; wal=null at 1732040937947 (+10 ms)Cleaning up temporary data from old regions at 1732040937950 (+3 ms)Region opened successfully at 1732040937958 (+8 ms) 2024-11-19T18:28:57,959 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-19T18:28:57,959 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9ae74df7e71a237c9c1e150fa4bda3ef, disabling compactions & flushes 2024-11-19T18:28:57,959 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732040937126.9ae74df7e71a237c9c1e150fa4bda3ef. 2024-11-19T18:28:57,959 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732040937126.9ae74df7e71a237c9c1e150fa4bda3ef. 2024-11-19T18:28:57,959 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732040937126.9ae74df7e71a237c9c1e150fa4bda3ef. after waiting 0 ms 2024-11-19T18:28:57,959 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732040937126.9ae74df7e71a237c9c1e150fa4bda3ef. 2024-11-19T18:28:57,961 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732040937126.9ae74df7e71a237c9c1e150fa4bda3ef. 2024-11-19T18:28:57,961 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9ae74df7e71a237c9c1e150fa4bda3ef: Waiting for close lock at 1732040937959Disabling compacts and flushes for region at 1732040937959Disabling writes for close at 1732040937959Writing region close event to WAL at 1732040937961 (+2 ms)Closed at 1732040937961 2024-11-19T18:28:57,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741870_1047 (size=93) 2024-11-19T18:28:57,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741870_1047 (size=93) 2024-11-19T18:28:57,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741870_1047 (size=93) 2024-11-19T18:28:57,969 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-19T18:28:57,969 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732040937684) 2024-11-19T18:28:57,985 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=404 (was 401) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:33478 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:33514 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:46906 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:35996 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=929 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=184 (was 184), ProcessCount=11 (was 11), AvailableMemoryMB=7293 (was 7417) 2024-11-19T18:28:57,997 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=404, OpenFileDescriptor=929, MaxFileDescriptor=1048576, SystemLoadAverage=184, ProcessCount=11, AvailableMemoryMB=7291 2024-11-19T18:28:58,014 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:28:58,020 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732040938013, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:28:58,021 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732040938021 2024-11-19T18:28:58,030 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732040938013/wal.1732040938021 2024-11-19T18:28:58,032 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:37769:37769)] 2024-11-19T18:28:58,037 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 89ee8070813e14b236f4511ca6b3b479, NAME => 'testSequentialEditLogSeqNum,,1732040938014.89ee8070813e14b236f4511ca6b3b479.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:28:58,038 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732040938014.89ee8070813e14b236f4511ca6b3b479.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:28:58,038 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 89ee8070813e14b236f4511ca6b3b479 2024-11-19T18:28:58,038 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 89ee8070813e14b236f4511ca6b3b479 2024-11-19T18:28:58,039 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:42267/hbase/data/default/testSequentialEditLogSeqNum/89ee8070813e14b236f4511ca6b3b479 doesn't exist for region: 89ee8070813e14b236f4511ca6b3b479 on table testSequentialEditLogSeqNum 2024-11-19T18:28:58,040 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 89ee8070813e14b236f4511ca6b3b479 on table testSequentialEditLogSeqNum 2024-11-19T18:28:58,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741875_1052 (size=62) 2024-11-19T18:28:58,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741875_1052 (size=62) 2024-11-19T18:28:58,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741875_1052 (size=62) 2024-11-19T18:28:58,054 INFO [StoreOpener-89ee8070813e14b236f4511ca6b3b479-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 89ee8070813e14b236f4511ca6b3b479 2024-11-19T18:28:58,055 INFO [StoreOpener-89ee8070813e14b236f4511ca6b3b479-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89ee8070813e14b236f4511ca6b3b479 columnFamilyName a 2024-11-19T18:28:58,056 DEBUG [StoreOpener-89ee8070813e14b236f4511ca6b3b479-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:58,056 INFO [StoreOpener-89ee8070813e14b236f4511ca6b3b479-1 {}] regionserver.HStore(327): Store=89ee8070813e14b236f4511ca6b3b479/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:58,056 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 89ee8070813e14b236f4511ca6b3b479 2024-11-19T18:28:58,057 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testSequentialEditLogSeqNum/89ee8070813e14b236f4511ca6b3b479 2024-11-19T18:28:58,058 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testSequentialEditLogSeqNum/89ee8070813e14b236f4511ca6b3b479 2024-11-19T18:28:58,058 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 89ee8070813e14b236f4511ca6b3b479 2024-11-19T18:28:58,058 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 89ee8070813e14b236f4511ca6b3b479 2024-11-19T18:28:58,060 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 89ee8070813e14b236f4511ca6b3b479 2024-11-19T18:28:58,063 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testSequentialEditLogSeqNum/89ee8070813e14b236f4511ca6b3b479/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T18:28:58,063 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 89ee8070813e14b236f4511ca6b3b479; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60235990, jitterRate=-0.10241380333900452}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T18:28:58,065 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 89ee8070813e14b236f4511ca6b3b479: Writing region info on filesystem at 1732040938038Initializing all the Stores at 1732040938053 (+15 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040938053Cleaning up temporary data from old regions at 1732040938058 (+5 ms)Region opened successfully at 1732040938065 (+7 ms) 2024-11-19T18:28:58,080 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 89ee8070813e14b236f4511ca6b3b479 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-19T18:28:58,101 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T18:28:58,102 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-19T18:28:58,103 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T18:28:58,103 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-19T18:28:58,104 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-19T18:28:58,104 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-19T18:28:58,105 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-19T18:28:58,105 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-19T18:28:58,105 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testSequentialEditLogSeqNum/89ee8070813e14b236f4511ca6b3b479/.tmp/a/4ca1267fa226459985071cf9b7085238 is 81, key is testSequentialEditLogSeqNum/a:x0/1732040938065/Put/seqid=0 2024-11-19T18:28:58,107 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-19T18:28:58,107 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-19T18:28:58,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741876_1053 (size=5833) 2024-11-19T18:28:58,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741876_1053 (size=5833) 2024-11-19T18:28:58,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741876_1053 (size=5833) 2024-11-19T18:28:58,115 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testSequentialEditLogSeqNum/89ee8070813e14b236f4511ca6b3b479/.tmp/a/4ca1267fa226459985071cf9b7085238 2024-11-19T18:28:58,122 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testSequentialEditLogSeqNum/89ee8070813e14b236f4511ca6b3b479/.tmp/a/4ca1267fa226459985071cf9b7085238 as hdfs://localhost:42267/hbase/data/default/testSequentialEditLogSeqNum/89ee8070813e14b236f4511ca6b3b479/a/4ca1267fa226459985071cf9b7085238 2024-11-19T18:28:58,129 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testSequentialEditLogSeqNum/89ee8070813e14b236f4511ca6b3b479/a/4ca1267fa226459985071cf9b7085238, entries=10, sequenceid=13, filesize=5.7 K 2024-11-19T18:28:58,130 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 89ee8070813e14b236f4511ca6b3b479 in 50ms, sequenceid=13, compaction requested=false 2024-11-19T18:28:58,131 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 89ee8070813e14b236f4511ca6b3b479: 2024-11-19T18:28:58,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T18:28:58,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T18:28:58,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T18:28:58,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T18:28:58,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T18:28:58,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741874_1051 (size=3017) 2024-11-19T18:28:58,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741874_1051 (size=3017) 2024-11-19T18:28:58,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741874_1051 (size=3017) 2024-11-19T18:28:58,160 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42267/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732040938013/wal.1732040938021, size=2.9 K (3017bytes) 2024-11-19T18:28:58,160 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42267/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732040938013/wal.1732040938021 2024-11-19T18:28:58,160 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42267/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732040938013/wal.1732040938021 after 0ms 2024-11-19T18:28:58,163 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42267/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732040938013/wal.1732040938021 took 4ms 2024-11-19T18:28:58,166 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42267/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732040938013/wal.1732040938021 so closing down 2024-11-19T18:28:58,166 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T18:28:58,167 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732040938021.temp 2024-11-19T18:28:58,168 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testSequentialEditLogSeqNum/89ee8070813e14b236f4511ca6b3b479/recovered.edits/0000000000000000003-wal.1732040938021.temp 2024-11-19T18:28:58,169 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T18:28:58,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741877_1054 (size=2433) 2024-11-19T18:28:58,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741877_1054 (size=2433) 2024-11-19T18:28:58,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741877_1054 (size=2433) 2024-11-19T18:28:58,180 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testSequentialEditLogSeqNum/89ee8070813e14b236f4511ca6b3b479/recovered.edits/0000000000000000003-wal.1732040938021.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-19T18:28:58,182 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42267/hbase/data/default/testSequentialEditLogSeqNum/89ee8070813e14b236f4511ca6b3b479/recovered.edits/0000000000000000003-wal.1732040938021.temp to hdfs://localhost:42267/hbase/data/default/testSequentialEditLogSeqNum/89ee8070813e14b236f4511ca6b3b479/recovered.edits/0000000000000000020 2024-11-19T18:28:58,182 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 18 ms; skipped=2; WAL=hdfs://localhost:42267/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732040938013/wal.1732040938021, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-11-19T18:28:58,182 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42267/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732040938013/wal.1732040938021, journal: Splitting hdfs://localhost:42267/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732040938013/wal.1732040938021, size=2.9 K (3017bytes) at 1732040938160Finishing writing output for hdfs://localhost:42267/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732040938013/wal.1732040938021 so closing down at 1732040938166 (+6 ms)Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testSequentialEditLogSeqNum/89ee8070813e14b236f4511ca6b3b479/recovered.edits/0000000000000000003-wal.1732040938021.temp at 1732040938168 (+2 ms)3 split writer threads finished at 1732040938169 (+1 ms)Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testSequentialEditLogSeqNum/89ee8070813e14b236f4511ca6b3b479/recovered.edits/0000000000000000003-wal.1732040938021.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732040938180 (+11 ms)Rename recovered edits hdfs://localhost:42267/hbase/data/default/testSequentialEditLogSeqNum/89ee8070813e14b236f4511ca6b3b479/recovered.edits/0000000000000000003-wal.1732040938021.temp to hdfs://localhost:42267/hbase/data/default/testSequentialEditLogSeqNum/89ee8070813e14b236f4511ca6b3b479/recovered.edits/0000000000000000020 at 1732040938182 (+2 ms)Processed 17 edits across 1 Regions in 18 ms; skipped=2; WAL=hdfs://localhost:42267/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732040938013/wal.1732040938021, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1732040938182 2024-11-19T18:28:58,197 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=409 (was 404) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:46906 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:35996 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=967 (was 929) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=184 (was 184), ProcessCount=11 (was 11), AvailableMemoryMB=7274 (was 7291) 2024-11-19T18:28:58,212 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=409, OpenFileDescriptor=967, MaxFileDescriptor=1048576, SystemLoadAverage=184, ProcessCount=11, AvailableMemoryMB=7273 2024-11-19T18:28:58,230 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:28:58,233 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:28:58,265 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T18:28:58,268 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-16346113, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/hregion-16346113, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:28:58,286 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-16346113/hregion-16346113.1732040938269, exclude list is [], retry=0 2024-11-19T18:28:58,289 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:58,290 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:58,290 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:58,294 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-16346113/hregion-16346113.1732040938269 2024-11-19T18:28:58,296 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:33801:33801)] 2024-11-19T18:28:58,296 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c4f04f271bfe469f401b6e79424e9526, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42267/hbase 2024-11-19T18:28:58,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741879_1056 (size=70) 2024-11-19T18:28:58,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741879_1056 (size=70) 2024-11-19T18:28:58,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741879_1056 (size=70) 2024-11-19T18:28:58,311 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:28:58,314 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,316 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4f04f271bfe469f401b6e79424e9526 columnFamilyName a 2024-11-19T18:28:58,316 DEBUG [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:58,317 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(327): Store=c4f04f271bfe469f401b6e79424e9526/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:58,317 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,319 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4f04f271bfe469f401b6e79424e9526 columnFamilyName b 2024-11-19T18:28:58,319 DEBUG [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:58,320 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(327): Store=c4f04f271bfe469f401b6e79424e9526/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:58,320 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,322 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4f04f271bfe469f401b6e79424e9526 columnFamilyName c 2024-11-19T18:28:58,322 DEBUG [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:58,322 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(327): Store=c4f04f271bfe469f401b6e79424e9526/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:58,322 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,323 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,324 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,325 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,325 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,325 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:28:58,327 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,329 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T18:28:58,330 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c4f04f271bfe469f401b6e79424e9526; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70960228, jitterRate=0.057389795780181885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:28:58,331 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c4f04f271bfe469f401b6e79424e9526: Writing region info on filesystem at 1732040938311Initializing all the Stores at 1732040938312 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040938312Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040938314 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040938314Cleaning up temporary data from old regions at 1732040938325 (+11 ms)Region opened successfully at 1732040938331 (+6 ms) 2024-11-19T18:28:58,332 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c4f04f271bfe469f401b6e79424e9526, disabling compactions & flushes 2024-11-19T18:28:58,332 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526. 2024-11-19T18:28:58,332 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526. 2024-11-19T18:28:58,332 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526. after waiting 0 ms 2024-11-19T18:28:58,332 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526. 2024-11-19T18:28:58,332 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526. 2024-11-19T18:28:58,332 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c4f04f271bfe469f401b6e79424e9526: Waiting for close lock at 1732040938331Disabling compacts and flushes for region at 1732040938331Disabling writes for close at 1732040938332 (+1 ms)Writing region close event to WAL at 1732040938332Closed at 1732040938332 2024-11-19T18:28:58,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741878_1055 (size=93) 2024-11-19T18:28:58,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741878_1055 (size=93) 2024-11-19T18:28:58,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741878_1055 (size=93) 2024-11-19T18:28:58,339 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-19T18:28:58,339 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-16346113:(num 1732040938269) 2024-11-19T18:28:58,339 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:28:58,341 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:28:58,357 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342, exclude list is [], retry=0 2024-11-19T18:28:58,360 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:28:58,361 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:28:58,361 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:28:58,363 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342 2024-11-19T18:28:58,363 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:40377:40377)] 2024-11-19T18:28:58,363 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c4f04f271bfe469f401b6e79424e9526, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:28:58,363 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:28:58,364 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,364 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,368 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,369 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4f04f271bfe469f401b6e79424e9526 columnFamilyName a 2024-11-19T18:28:58,369 DEBUG [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:58,370 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(327): Store=c4f04f271bfe469f401b6e79424e9526/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:58,370 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,371 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4f04f271bfe469f401b6e79424e9526 columnFamilyName b 2024-11-19T18:28:58,371 DEBUG [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:58,371 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(327): Store=c4f04f271bfe469f401b6e79424e9526/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:58,371 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,372 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4f04f271bfe469f401b6e79424e9526 columnFamilyName c 2024-11-19T18:28:58,372 DEBUG [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:28:58,372 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(327): Store=c4f04f271bfe469f401b6e79424e9526/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:28:58,373 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,373 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,374 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,375 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,375 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,376 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:28:58,377 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:28:58,378 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c4f04f271bfe469f401b6e79424e9526; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73040840, jitterRate=0.08839333057403564}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:28:58,379 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c4f04f271bfe469f401b6e79424e9526: Writing region info on filesystem at 1732040938364Initializing all the Stores at 1732040938365 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040938365Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040938368 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040938368Cleaning up temporary data from old regions at 1732040938375 (+7 ms)Region opened successfully at 1732040938379 (+4 ms) 2024-11-19T18:28:58,384 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732040938382/Put/seqid=0 2024-11-19T18:28:58,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741881_1058 (size=4826) 2024-11-19T18:28:58,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741881_1058 (size=4826) 2024-11-19T18:28:58,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741881_1058 (size=4826) 2024-11-19T18:28:58,395 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:42267/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in c4f04f271bfe469f401b6e79424e9526/a 2024-11-19T18:28:58,402 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-19T18:28:58,403 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-19T18:28:58,403 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c4f04f271bfe469f401b6e79424e9526: 2024-11-19T18:28:58,404 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/a/723415ad2c02438db4b06ee58985bb33_SeqId_3_ 2024-11-19T18:28:58,405 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:42267/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into c4f04f271bfe469f401b6e79424e9526/a as hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/a/723415ad2c02438db4b06ee58985bb33_SeqId_3_ - updating store file list. 2024-11-19T18:28:58,411 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 723415ad2c02438db4b06ee58985bb33_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-19T18:28:58,411 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/a/723415ad2c02438db4b06ee58985bb33_SeqId_3_ into c4f04f271bfe469f401b6e79424e9526/a 2024-11-19T18:28:58,411 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:42267/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into c4f04f271bfe469f401b6e79424e9526/a (new location: hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/a/723415ad2c02438db4b06ee58985bb33_SeqId_3_) 2024-11-19T18:28:58,461 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42267/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342, size=0 (0bytes) 2024-11-19T18:28:58,461 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:42267/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342 might be still open, length is 0 2024-11-19T18:28:58,462 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42267/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342 2024-11-19T18:28:58,462 WARN [IPC Server handler 0 on default port 42267 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-19T18:28:58,462 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42267/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342 after 0ms 2024-11-19T18:28:58,620 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:46998 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33471:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46998 dst: /127.0.0.1:33471 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33471 remote=/127.0.0.1:46998]. Total timeout mills is 60000, 59798 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:28:58,621 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:33690 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33841:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33690 dst: /127.0.0.1:33841 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:28:58,621 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:36118 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41941:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36118 dst: /127.0.0.1:41941 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:28:58,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741880_1059 (size=504) 2024-11-19T18:28:58,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741880_1059 (size=504) 2024-11-19T18:29:02,463 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42267/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342 after 4001ms 2024-11-19T18:29:02,467 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42267/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342 took 4006ms 2024-11-19T18:29:02,469 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:42267/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342; continuing. 2024-11-19T18:29:02,469 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42267/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342 so closing down 2024-11-19T18:29:02,469 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T18:29:02,471 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732040938342.temp 2024-11-19T18:29:02,473 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/recovered.edits/0000000000000000005-wal.1732040938342.temp 2024-11-19T18:29:02,473 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T18:29:02,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741882_1060 (size=238) 2024-11-19T18:29:02,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741882_1060 (size=238) 2024-11-19T18:29:02,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741882_1060 (size=238) 2024-11-19T18:29:02,482 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/recovered.edits/0000000000000000005-wal.1732040938342.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-19T18:29:02,483 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/recovered.edits/0000000000000000005-wal.1732040938342.temp to hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/recovered.edits/0000000000000000005 2024-11-19T18:29:02,484 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 17 ms; skipped=1; WAL=hdfs://localhost:42267/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342, size=0, length=0, corrupted=false, cancelled=false 2024-11-19T18:29:02,484 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42267/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342, journal: Splitting hdfs://localhost:42267/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342, size=0 (0bytes) at 1732040938461Finishing writing output for hdfs://localhost:42267/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342 so closing down at 1732040942469 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/recovered.edits/0000000000000000005-wal.1732040938342.temp at 1732040942473 (+4 ms)3 split writer threads finished at 1732040942473Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/recovered.edits/0000000000000000005-wal.1732040938342.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732040942482 (+9 ms)Rename recovered edits hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/recovered.edits/0000000000000000005-wal.1732040938342.temp to hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/recovered.edits/0000000000000000005 at 1732040942484 (+2 ms)Processed 2 edits across 1 Regions in 17 ms; skipped=1; WAL=hdfs://localhost:42267/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342, size=0, length=0, corrupted=false, cancelled=false at 1732040942484 2024-11-19T18:29:02,486 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42267/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342 to hdfs://localhost:42267/hbase/oldWALs/wal.1732040938342 2024-11-19T18:29:02,487 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/recovered.edits/0000000000000000005 2024-11-19T18:29:02,487 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:29:02,489 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:29:02,504 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040942489, exclude list is [], retry=0 2024-11-19T18:29:02,506 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:29:02,507 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:29:02,507 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:29:02,509 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040942489 2024-11-19T18:29:02,509 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:37769:37769)] 2024-11-19T18:29:02,510 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c4f04f271bfe469f401b6e79424e9526, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:02,510 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:02,510 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:29:02,510 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:29:02,512 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:29:02,513 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4f04f271bfe469f401b6e79424e9526 columnFamilyName a 2024-11-19T18:29:02,513 DEBUG [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:02,519 DEBUG [StoreFileOpener-c4f04f271bfe469f401b6e79424e9526-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 723415ad2c02438db4b06ee58985bb33_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-19T18:29:02,519 DEBUG [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/a/723415ad2c02438db4b06ee58985bb33_SeqId_3_ 2024-11-19T18:29:02,519 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(327): Store=c4f04f271bfe469f401b6e79424e9526/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:02,519 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:29:02,521 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4f04f271bfe469f401b6e79424e9526 columnFamilyName b 2024-11-19T18:29:02,521 DEBUG [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:02,521 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(327): Store=c4f04f271bfe469f401b6e79424e9526/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:02,521 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:29:02,522 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4f04f271bfe469f401b6e79424e9526 columnFamilyName c 2024-11-19T18:29:02,522 DEBUG [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:02,523 INFO [StoreOpener-c4f04f271bfe469f401b6e79424e9526-1 {}] regionserver.HStore(327): Store=c4f04f271bfe469f401b6e79424e9526/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:02,523 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:29:02,524 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:29:02,526 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:29:02,526 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/recovered.edits/0000000000000000005 2024-11-19T18:29:02,530 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/recovered.edits/0000000000000000005 2024-11-19T18:29:02,530 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c4f04f271bfe469f401b6e79424e9526 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-19T18:29:02,546 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/.tmp/a/a0bea81058654935ab43845d87715ed6 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732040938416/Put/seqid=0 2024-11-19T18:29:02,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741884_1062 (size=5149) 2024-11-19T18:29:02,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741884_1062 (size=5149) 2024-11-19T18:29:02,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741884_1062 (size=5149) 2024-11-19T18:29:02,554 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/.tmp/a/a0bea81058654935ab43845d87715ed6 2024-11-19T18:29:02,560 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/.tmp/a/a0bea81058654935ab43845d87715ed6 as hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/a/a0bea81058654935ab43845d87715ed6 2024-11-19T18:29:02,567 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/a/a0bea81058654935ab43845d87715ed6, entries=1, sequenceid=5, filesize=5.0 K 2024-11-19T18:29:02,567 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for c4f04f271bfe469f401b6e79424e9526 in 37ms, sequenceid=5, compaction requested=false; wal=null 2024-11-19T18:29:02,568 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/recovered.edits/0000000000000000005 2024-11-19T18:29:02,569 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:29:02,569 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:29:02,570 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:29:02,572 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c4f04f271bfe469f401b6e79424e9526 2024-11-19T18:29:02,574 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c4f04f271bfe469f401b6e79424e9526/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-19T18:29:02,575 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c4f04f271bfe469f401b6e79424e9526; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67924104, jitterRate=0.012148022651672363}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:29:02,575 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c4f04f271bfe469f401b6e79424e9526: Writing region info on filesystem at 1732040942510Initializing all the Stores at 1732040942511 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040942511Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040942511Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040942511Obtaining lock to block concurrent updates at 1732040942530 (+19 ms)Preparing flush snapshotting stores in c4f04f271bfe469f401b6e79424e9526 at 1732040942530Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732040942530Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526. at 1732040942530Flushing c4f04f271bfe469f401b6e79424e9526/a: creating writer at 1732040942530Flushing c4f04f271bfe469f401b6e79424e9526/a: appending metadata at 1732040942546 (+16 ms)Flushing c4f04f271bfe469f401b6e79424e9526/a: closing flushed file at 1732040942546Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@266cdd10: reopening flushed file at 1732040942560 (+14 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for c4f04f271bfe469f401b6e79424e9526 in 37ms, sequenceid=5, compaction requested=false; wal=null at 1732040942567 (+7 ms)Cleaning up temporary data from old regions at 1732040942569 (+2 ms)Region opened successfully at 1732040942575 (+6 ms) 2024-11-19T18:29:02,579 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c4f04f271bfe469f401b6e79424e9526, disabling compactions & flushes 2024-11-19T18:29:02,580 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526. 2024-11-19T18:29:02,580 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526. 2024-11-19T18:29:02,580 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526. after waiting 0 ms 2024-11-19T18:29:02,580 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526. 2024-11-19T18:29:02,581 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732040938231.c4f04f271bfe469f401b6e79424e9526. 2024-11-19T18:29:02,581 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c4f04f271bfe469f401b6e79424e9526: Waiting for close lock at 1732040942579Disabling compacts and flushes for region at 1732040942579Disabling writes for close at 1732040942580 (+1 ms)Writing region close event to WAL at 1732040942581 (+1 ms)Closed at 1732040942581 2024-11-19T18:29:02,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741883_1061 (size=93) 2024-11-19T18:29:02,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741883_1061 (size=93) 2024-11-19T18:29:02,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741883_1061 (size=93) 2024-11-19T18:29:02,587 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-19T18:29:02,587 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732040942489) 2024-11-19T18:29:02,604 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=412 (was 409) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:42267 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:42267 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_630438918_22 at /127.0.0.1:52586 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_630438918_22 at /127.0.0.1:51230 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_630438918_22 at /127.0.0.1:55538 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1025 (was 967) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=177 (was 184), ProcessCount=11 (was 11), AvailableMemoryMB=7183 (was 7273) 2024-11-19T18:29:02,607 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-19T18:29:02,623 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=412, OpenFileDescriptor=1025, MaxFileDescriptor=1048576, SystemLoadAverage=177, ProcessCount=11, AvailableMemoryMB=7176 2024-11-19T18:29:02,623 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-19T18:29:02,640 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:29:02,646 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T18:29:02,651 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 30db5f576be8,42887,1732040927686 2024-11-19T18:29:02,654 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2f78862a 2024-11-19T18:29:02,655 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T18:29:02,657 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T18:29:02,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42887 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T18:29:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42887 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-19T18:29:02,670 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T18:29:02,672 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42887 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-19T18:29:02,672 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:02,674 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T18:29:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42887 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T18:29:02,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741885_1063 (size=694) 2024-11-19T18:29:02,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741885_1063 (size=694) 2024-11-19T18:29:02,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741885_1063 (size=694) 2024-11-19T18:29:02,691 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 74fc47bc7ee81b8f5f6b4b68f2317277, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01 2024-11-19T18:29:02,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741886_1064 (size=77) 2024-11-19T18:29:02,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741886_1064 (size=77) 2024-11-19T18:29:02,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741886_1064 (size=77) 2024-11-19T18:29:02,705 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:02,705 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 74fc47bc7ee81b8f5f6b4b68f2317277, disabling compactions & flushes 2024-11-19T18:29:02,705 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:02,706 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:02,706 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. after waiting 0 ms 2024-11-19T18:29:02,706 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:02,706 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:02,706 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 74fc47bc7ee81b8f5f6b4b68f2317277: Waiting for close lock at 1732040942705Disabling compacts and flushes for region at 1732040942705Disabling writes for close at 1732040942706 (+1 ms)Writing region close event to WAL at 1732040942706Closed at 1732040942706 2024-11-19T18:29:02,708 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T18:29:02,713 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732040942708"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732040942708"}]},"ts":"1732040942708"} 2024-11-19T18:29:02,716 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T18:29:02,718 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T18:29:02,721 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732040942718"}]},"ts":"1732040942718"} 2024-11-19T18:29:02,725 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-19T18:29:02,725 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(204): Hosts are {30db5f576be8=0} racks are {/default-rack=0} 2024-11-19T18:29:02,727 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T18:29:02,727 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T18:29:02,727 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-19T18:29:02,727 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T18:29:02,727 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T18:29:02,727 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-19T18:29:02,727 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T18:29:02,727 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T18:29:02,727 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-19T18:29:02,727 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T18:29:02,728 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, ASSIGN}] 2024-11-19T18:29:02,730 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, ASSIGN 2024-11-19T18:29:02,732 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, ASSIGN; state=OFFLINE, location=30db5f576be8,41975,1732040928526; forceNewPlan=false, retain=false 2024-11-19T18:29:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42887 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T18:29:02,885 INFO [30db5f576be8:42887 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-19T18:29:02,886 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=74fc47bc7ee81b8f5f6b4b68f2317277, regionState=OPENING, regionLocation=30db5f576be8,41975,1732040928526 2024-11-19T18:29:02,890 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, ASSIGN because future has completed 2024-11-19T18:29:02,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,41975,1732040928526}] 2024-11-19T18:29:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42887 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T18:29:03,044 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T18:29:03,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42849, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T18:29:03,052 INFO [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:03,053 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 74fc47bc7ee81b8f5f6b4b68f2317277, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:03,053 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,053 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:03,053 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,053 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,055 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,057 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74fc47bc7ee81b8f5f6b4b68f2317277 columnFamilyName cf1 2024-11-19T18:29:03,057 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:03,058 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(327): Store=74fc47bc7ee81b8f5f6b4b68f2317277/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:03,058 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,059 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74fc47bc7ee81b8f5f6b4b68f2317277 columnFamilyName cf2 2024-11-19T18:29:03,059 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:03,060 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(327): Store=74fc47bc7ee81b8f5f6b4b68f2317277/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:03,060 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,061 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,061 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,062 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,062 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,063 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-19T18:29:03,065 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,068 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T18:29:03,069 INFO [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 74fc47bc7ee81b8f5f6b4b68f2317277; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74565827, jitterRate=0.1111174076795578}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-19T18:29:03,069 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,070 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 74fc47bc7ee81b8f5f6b4b68f2317277: Running coprocessor pre-open hook at 1732040943053Writing region info on filesystem at 1732040943053Initializing all the Stores at 1732040943055 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040943055Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040943055Cleaning up temporary data from old regions at 1732040943062 (+7 ms)Running coprocessor post-open hooks at 1732040943069 (+7 ms)Region opened successfully at 1732040943070 (+1 ms) 2024-11-19T18:29:03,072 INFO [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., pid=6, masterSystemTime=1732040943044 2024-11-19T18:29:03,076 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:03,076 INFO [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:03,077 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=74fc47bc7ee81b8f5f6b4b68f2317277, regionState=OPEN, openSeqNum=2, regionLocation=30db5f576be8,41975,1732040928526 2024-11-19T18:29:03,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,41975,1732040928526 because future has completed 2024-11-19T18:29:03,086 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T18:29:03,086 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,41975,1732040928526 in 192 msec 2024-11-19T18:29:03,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T18:29:03,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, ASSIGN in 358 msec 2024-11-19T18:29:03,090 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T18:29:03,090 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732040943090"}]},"ts":"1732040943090"} 2024-11-19T18:29:03,093 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-19T18:29:03,094 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T18:29:03,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 431 msec 2024-11-19T18:29:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42887 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T18:29:03,310 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-19T18:29:03,310 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-19T18:29:03,312 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T18:29:03,317 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-19T18:29:03,317 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T18:29:03,318 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-19T18:29:03,330 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=2] 2024-11-19T18:29:03,331 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T18:29:03,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T18:29:03,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42887 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=74fc47bc7ee81b8f5f6b4b68f2317277, source=30db5f576be8,41975,1732040928526, destination=30db5f576be8,38055,1732040928383, warming up region on 30db5f576be8,38055,1732040928383 2024-11-19T18:29:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42887 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T18:29:03,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42887 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=74fc47bc7ee81b8f5f6b4b68f2317277, source=30db5f576be8,41975,1732040928526, destination=30db5f576be8,38055,1732040928383, running balancer 2024-11-19T18:29:03,381 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50563, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T18:29:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42887 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, REOPEN/MOVE 2024-11-19T18:29:03,382 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, REOPEN/MOVE 2024-11-19T18:29:03,384 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=74fc47bc7ee81b8f5f6b4b68f2317277, regionState=CLOSING, regionLocation=30db5f576be8,41975,1732040928526 2024-11-19T18:29:03,386 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38055 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38055 {}] regionserver.HRegion(7855): Warmup {ENCODED => 74fc47bc7ee81b8f5f6b4b68f2317277, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38055 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:03,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, REOPEN/MOVE because future has completed 2024-11-19T18:29:03,387 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,388 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T18:29:03,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,41975,1732040928526}] 2024-11-19T18:29:03,389 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74fc47bc7ee81b8f5f6b4b68f2317277 columnFamilyName cf1 2024-11-19T18:29:03,389 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:03,390 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(327): Store=74fc47bc7ee81b8f5f6b4b68f2317277/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:03,390 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,391 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74fc47bc7ee81b8f5f6b4b68f2317277 columnFamilyName cf2 2024-11-19T18:29:03,391 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:03,391 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(327): Store=74fc47bc7ee81b8f5f6b4b68f2317277/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38055 {}] regionserver.HRegion(1722): Closing 74fc47bc7ee81b8f5f6b4b68f2317277, disabling compactions & flushes 2024-11-19T18:29:03,392 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38055 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38055 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38055 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. after waiting 0 ms 2024-11-19T18:29:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38055 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:03,392 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38055 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38055 {}] regionserver.HRegion(1676): Region close journal for 74fc47bc7ee81b8f5f6b4b68f2317277: Waiting for close lock at 1732040943391Disabling compacts and flushes for region at 1732040943391Disabling writes for close at 1732040943392 (+1 ms)Writing region close event to WAL at 1732040943392Closed at 1732040943392 2024-11-19T18:29:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42887 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-19T18:29:03,546 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,547 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T18:29:03,547 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 74fc47bc7ee81b8f5f6b4b68f2317277, disabling compactions & flushes 2024-11-19T18:29:03,547 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:03,547 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:03,547 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. after waiting 0 ms 2024-11-19T18:29:03,547 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:03,547 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 74fc47bc7ee81b8f5f6b4b68f2317277 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-19T18:29:03,567 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/.tmp/cf1/e3f1bea401814c04b8531d281803caec is 35, key is r1/cf1:q/1732040943360/Put/seqid=0 2024-11-19T18:29:03,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741887_1065 (size=4783) 2024-11-19T18:29:03,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741887_1065 (size=4783) 2024-11-19T18:29:03,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741887_1065 (size=4783) 2024-11-19T18:29:03,575 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/.tmp/cf1/e3f1bea401814c04b8531d281803caec 2024-11-19T18:29:03,583 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/.tmp/cf1/e3f1bea401814c04b8531d281803caec as hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/e3f1bea401814c04b8531d281803caec 2024-11-19T18:29:03,589 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/e3f1bea401814c04b8531d281803caec, entries=1, sequenceid=5, filesize=4.7 K 2024-11-19T18:29:03,591 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 74fc47bc7ee81b8f5f6b4b68f2317277 in 43ms, sequenceid=5, compaction requested=false 2024-11-19T18:29:03,591 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-19T18:29:03,596 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-19T18:29:03,598 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:03,599 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 74fc47bc7ee81b8f5f6b4b68f2317277: Waiting for close lock at 1732040943547Running coprocessor pre-close hooks at 1732040943547Disabling compacts and flushes for region at 1732040943547Disabling writes for close at 1732040943547Obtaining lock to block concurrent updates at 1732040943547Preparing flush snapshotting stores in 74fc47bc7ee81b8f5f6b4b68f2317277 at 1732040943547Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732040943548 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. at 1732040943549 (+1 ms)Flushing 74fc47bc7ee81b8f5f6b4b68f2317277/cf1: creating writer at 1732040943549Flushing 74fc47bc7ee81b8f5f6b4b68f2317277/cf1: appending metadata at 1732040943567 (+18 ms)Flushing 74fc47bc7ee81b8f5f6b4b68f2317277/cf1: closing flushed file at 1732040943567Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b19a81a: reopening flushed file at 1732040943582 (+15 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 74fc47bc7ee81b8f5f6b4b68f2317277 in 43ms, sequenceid=5, compaction requested=false at 1732040943591 (+9 ms)Writing region close event to WAL at 1732040943592 (+1 ms)Running coprocessor post-close hooks at 1732040943596 (+4 ms)Closed at 1732040943598 (+2 ms) 2024-11-19T18:29:03,599 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 74fc47bc7ee81b8f5f6b4b68f2317277 move to 30db5f576be8,38055,1732040928383 record at close sequenceid=5 2024-11-19T18:29:03,602 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,603 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=74fc47bc7ee81b8f5f6b4b68f2317277, regionState=CLOSED 2024-11-19T18:29:03,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,41975,1732040928526 because future has completed 2024-11-19T18:29:03,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T18:29:03,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,41975,1732040928526 in 219 msec 2024-11-19T18:29:03,611 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, REOPEN/MOVE; state=CLOSED, location=30db5f576be8,38055,1732040928383; forceNewPlan=false, retain=false 2024-11-19T18:29:03,762 INFO [30db5f576be8:42887 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-19T18:29:03,762 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=74fc47bc7ee81b8f5f6b4b68f2317277, regionState=OPENING, regionLocation=30db5f576be8,38055,1732040928383 2024-11-19T18:29:03,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, REOPEN/MOVE because future has completed 2024-11-19T18:29:03,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,38055,1732040928383}] 2024-11-19T18:29:03,923 INFO [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:03,923 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 74fc47bc7ee81b8f5f6b4b68f2317277, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:03,924 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,924 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:03,924 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,924 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,928 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,929 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74fc47bc7ee81b8f5f6b4b68f2317277 columnFamilyName cf1 2024-11-19T18:29:03,929 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:03,937 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/e3f1bea401814c04b8531d281803caec 2024-11-19T18:29:03,937 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(327): Store=74fc47bc7ee81b8f5f6b4b68f2317277/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:03,938 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,939 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74fc47bc7ee81b8f5f6b4b68f2317277 columnFamilyName cf2 2024-11-19T18:29:03,939 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:03,940 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(327): Store=74fc47bc7ee81b8f5f6b4b68f2317277/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:03,940 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,941 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,942 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,942 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,943 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,943 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-19T18:29:03,945 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,946 INFO [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 74fc47bc7ee81b8f5f6b4b68f2317277; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67734102, jitterRate=0.009316772222518921}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-19T18:29:03,946 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:03,947 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 74fc47bc7ee81b8f5f6b4b68f2317277: Running coprocessor pre-open hook at 1732040943924Writing region info on filesystem at 1732040943924Initializing all the Stores at 1732040943927 (+3 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040943927Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040943927Cleaning up temporary data from old regions at 1732040943943 (+16 ms)Running coprocessor post-open hooks at 1732040943946 (+3 ms)Region opened successfully at 1732040943946 2024-11-19T18:29:03,948 INFO [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., pid=9, masterSystemTime=1732040943918 2024-11-19T18:29:03,951 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:03,951 INFO [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:03,952 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=74fc47bc7ee81b8f5f6b4b68f2317277, regionState=OPEN, openSeqNum=9, regionLocation=30db5f576be8,38055,1732040928383 2024-11-19T18:29:03,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,38055,1732040928383 because future has completed 2024-11-19T18:29:03,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-19T18:29:03,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,38055,1732040928383 in 190 msec 2024-11-19T18:29:03,961 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, REOPEN/MOVE in 579 msec 2024-11-19T18:29:03,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T18:29:03,990 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T18:29:03,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41975 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:44568 deadline: 1732041003995, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=30db5f576be8 port=38055 startCode=1732040928383. As of locationSeqNum=5. 2024-11-19T18:29:04,002 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=30db5f576be8 port=38055 startCode=1732040928383. As of locationSeqNum=5. 2024-11-19T18:29:04,003 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=30db5f576be8 port=38055 startCode=1732040928383. As of locationSeqNum=5. 2024-11-19T18:29:04,003 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,38055,1732040928383, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=30db5f576be8 port=38055 startCode=1732040928383. As of locationSeqNum=5. 2024-11-19T18:29:04,110 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T18:29:04,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T18:29:04,124 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 74fc47bc7ee81b8f5f6b4b68f2317277 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-19T18:29:04,144 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/.tmp/cf1/8ef36f89b2bc4161919a03d4d544444e is 29, key is r1/cf1:/1732040944113/DeleteFamily/seqid=0 2024-11-19T18:29:04,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741888_1066 (size=4906) 2024-11-19T18:29:04,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741888_1066 (size=4906) 2024-11-19T18:29:04,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741888_1066 (size=4906) 2024-11-19T18:29:04,153 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/.tmp/cf1/8ef36f89b2bc4161919a03d4d544444e 2024-11-19T18:29:04,160 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8ef36f89b2bc4161919a03d4d544444e 2024-11-19T18:29:04,176 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/.tmp/cf2/8a3c2e4b0d224864af26cc857ffcc786 is 29, key is r1/cf2:/1732040944113/DeleteFamily/seqid=0 2024-11-19T18:29:04,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741889_1067 (size=4906) 2024-11-19T18:29:04,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741889_1067 (size=4906) 2024-11-19T18:29:04,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741889_1067 (size=4906) 2024-11-19T18:29:04,186 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/.tmp/cf2/8a3c2e4b0d224864af26cc857ffcc786 2024-11-19T18:29:04,192 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8a3c2e4b0d224864af26cc857ffcc786 2024-11-19T18:29:04,193 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/.tmp/cf1/8ef36f89b2bc4161919a03d4d544444e as hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/8ef36f89b2bc4161919a03d4d544444e 2024-11-19T18:29:04,199 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8ef36f89b2bc4161919a03d4d544444e 2024-11-19T18:29:04,200 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/8ef36f89b2bc4161919a03d4d544444e, entries=1, sequenceid=12, filesize=4.8 K 2024-11-19T18:29:04,201 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/.tmp/cf2/8a3c2e4b0d224864af26cc857ffcc786 as hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf2/8a3c2e4b0d224864af26cc857ffcc786 2024-11-19T18:29:04,207 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8a3c2e4b0d224864af26cc857ffcc786 2024-11-19T18:29:04,207 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf2/8a3c2e4b0d224864af26cc857ffcc786, entries=1, sequenceid=12, filesize=4.8 K 2024-11-19T18:29:04,209 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 74fc47bc7ee81b8f5f6b4b68f2317277 in 85ms, sequenceid=12, compaction requested=false 2024-11-19T18:29:04,209 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 74fc47bc7ee81b8f5f6b4b68f2317277: 2024-11-19T18:29:04,212 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-19T18:29:04,213 DEBUG [Time-limited test {}] regionserver.HStore(1541): 74fc47bc7ee81b8f5f6b4b68f2317277/cf1 is initiating major compaction (all files) 2024-11-19T18:29:04,214 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T18:29:04,214 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T18:29:04,214 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 74fc47bc7ee81b8f5f6b4b68f2317277/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,215 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/e3f1bea401814c04b8531d281803caec, hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/8ef36f89b2bc4161919a03d4d544444e] into tmpdir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/.tmp, totalSize=9.5 K 2024-11-19T18:29:04,216 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e3f1bea401814c04b8531d281803caec, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732040943360 2024-11-19T18:29:04,216 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8ef36f89b2bc4161919a03d4d544444e, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-19T18:29:04,229 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 74fc47bc7ee81b8f5f6b4b68f2317277#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T18:29:04,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741890_1068 (size=4626) 2024-11-19T18:29:04,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741890_1068 (size=4626) 2024-11-19T18:29:04,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741890_1068 (size=4626) 2024-11-19T18:29:04,248 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/.tmp/cf1/09bd95c77fec4137914dad15ca93e0c6 as hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/09bd95c77fec4137914dad15ca93e0c6 2024-11-19T18:29:04,266 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 74fc47bc7ee81b8f5f6b4b68f2317277/cf1 of 74fc47bc7ee81b8f5f6b4b68f2317277 into 09bd95c77fec4137914dad15ca93e0c6(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T18:29:04,266 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 74fc47bc7ee81b8f5f6b4b68f2317277: 2024-11-19T18:29:04,266 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-19T18:29:04,266 DEBUG [Time-limited test {}] regionserver.HStore(1541): 74fc47bc7ee81b8f5f6b4b68f2317277/cf2 is initiating major compaction (all files) 2024-11-19T18:29:04,267 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T18:29:04,267 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T18:29:04,267 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 74fc47bc7ee81b8f5f6b4b68f2317277/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,267 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf2/8a3c2e4b0d224864af26cc857ffcc786] into tmpdir=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/.tmp, totalSize=4.8 K 2024-11-19T18:29:04,268 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8a3c2e4b0d224864af26cc857ffcc786, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-19T18:29:04,274 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 74fc47bc7ee81b8f5f6b4b68f2317277#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T18:29:04,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741891_1069 (size=4592) 2024-11-19T18:29:04,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741891_1069 (size=4592) 2024-11-19T18:29:04,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741891_1069 (size=4592) 2024-11-19T18:29:04,291 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/.tmp/cf2/d347fe4fb1cb4231a911dcab472bde79 as hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf2/d347fe4fb1cb4231a911dcab472bde79 2024-11-19T18:29:04,298 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 74fc47bc7ee81b8f5f6b4b68f2317277/cf2 of 74fc47bc7ee81b8f5f6b4b68f2317277 into d347fe4fb1cb4231a911dcab472bde79(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T18:29:04,298 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 74fc47bc7ee81b8f5f6b4b68f2317277: 2024-11-19T18:29:04,303 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42887 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=74fc47bc7ee81b8f5f6b4b68f2317277, source=30db5f576be8,38055,1732040928383, destination=30db5f576be8,41975,1732040928526, warming up region on 30db5f576be8,41975,1732040928526 2024-11-19T18:29:04,303 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42887 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=74fc47bc7ee81b8f5f6b4b68f2317277, source=30db5f576be8,38055,1732040928383, destination=30db5f576be8,41975,1732040928526, running balancer 2024-11-19T18:29:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42887 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, REOPEN/MOVE 2024-11-19T18:29:04,306 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, REOPEN/MOVE 2024-11-19T18:29:04,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41975 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41975 {}] regionserver.HRegion(7855): Warmup {ENCODED => 74fc47bc7ee81b8f5f6b4b68f2317277, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:04,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41975 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:04,307 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=74fc47bc7ee81b8f5f6b4b68f2317277, regionState=CLOSING, regionLocation=30db5f576be8,38055,1732040928383 2024-11-19T18:29:04,308 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,309 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74fc47bc7ee81b8f5f6b4b68f2317277 columnFamilyName cf1 2024-11-19T18:29:04,309 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:04,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, REOPEN/MOVE because future has completed 2024-11-19T18:29:04,311 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T18:29:04,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,38055,1732040928383}] 2024-11-19T18:29:04,319 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/09bd95c77fec4137914dad15ca93e0c6 2024-11-19T18:29:04,324 INFO [StoreFileOpener-74fc47bc7ee81b8f5f6b4b68f2317277-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8ef36f89b2bc4161919a03d4d544444e 2024-11-19T18:29:04,324 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/8ef36f89b2bc4161919a03d4d544444e 2024-11-19T18:29:04,329 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/e3f1bea401814c04b8531d281803caec 2024-11-19T18:29:04,329 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(327): Store=74fc47bc7ee81b8f5f6b4b68f2317277/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:04,330 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,331 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74fc47bc7ee81b8f5f6b4b68f2317277 columnFamilyName cf2 2024-11-19T18:29:04,331 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:04,338 INFO [StoreFileOpener-74fc47bc7ee81b8f5f6b4b68f2317277-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8a3c2e4b0d224864af26cc857ffcc786 2024-11-19T18:29:04,339 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf2/8a3c2e4b0d224864af26cc857ffcc786 2024-11-19T18:29:04,343 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf2/d347fe4fb1cb4231a911dcab472bde79 2024-11-19T18:29:04,344 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(327): Store=74fc47bc7ee81b8f5f6b4b68f2317277/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41975 {}] regionserver.HRegion(1722): Closing 74fc47bc7ee81b8f5f6b4b68f2317277, disabling compactions & flushes 2024-11-19T18:29:04,344 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41975 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41975 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41975 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. after waiting 0 ms 2024-11-19T18:29:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41975 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,346 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41975 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41975 {}] regionserver.HRegion(1676): Region close journal for 74fc47bc7ee81b8f5f6b4b68f2317277: Waiting for close lock at 1732040944344Disabling compacts and flushes for region at 1732040944344Disabling writes for close at 1732040944344Writing region close event to WAL at 1732040944345 (+1 ms)Closed at 1732040944345 2024-11-19T18:29:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42887 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-19T18:29:04,466 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,466 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T18:29:04,466 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 74fc47bc7ee81b8f5f6b4b68f2317277, disabling compactions & flushes 2024-11-19T18:29:04,466 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,466 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,466 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. after waiting 0 ms 2024-11-19T18:29:04,466 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,467 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/e3f1bea401814c04b8531d281803caec, hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/8ef36f89b2bc4161919a03d4d544444e] to archive 2024-11-19T18:29:04,470 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T18:29:04,475 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/e3f1bea401814c04b8531d281803caec to hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/e3f1bea401814c04b8531d281803caec 2024-11-19T18:29:04,477 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/8ef36f89b2bc4161919a03d4d544444e to hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/8ef36f89b2bc4161919a03d4d544444e 2024-11-19T18:29:04,494 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf2/8a3c2e4b0d224864af26cc857ffcc786] to archive 2024-11-19T18:29:04,495 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T18:29:04,498 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf2/8a3c2e4b0d224864af26cc857ffcc786 to hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf2/8a3c2e4b0d224864af26cc857ffcc786 2024-11-19T18:29:04,505 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-19T18:29:04,506 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,506 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 74fc47bc7ee81b8f5f6b4b68f2317277: Waiting for close lock at 1732040944466Running coprocessor pre-close hooks at 1732040944466Disabling compacts and flushes for region at 1732040944466Disabling writes for close at 1732040944466Writing region close event to WAL at 1732040944500 (+34 ms)Running coprocessor post-close hooks at 1732040944506 (+6 ms)Closed at 1732040944506 2024-11-19T18:29:04,506 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 74fc47bc7ee81b8f5f6b4b68f2317277 move to 30db5f576be8,41975,1732040928526 record at close sequenceid=12 2024-11-19T18:29:04,509 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,511 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=74fc47bc7ee81b8f5f6b4b68f2317277, regionState=CLOSED 2024-11-19T18:29:04,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,38055,1732040928383 because future has completed 2024-11-19T18:29:04,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-19T18:29:04,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,38055,1732040928383 in 205 msec 2024-11-19T18:29:04,519 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, REOPEN/MOVE; state=CLOSED, location=30db5f576be8,41975,1732040928526; forceNewPlan=false, retain=false 2024-11-19T18:29:04,670 INFO [30db5f576be8:42887 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-19T18:29:04,670 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=74fc47bc7ee81b8f5f6b4b68f2317277, regionState=OPENING, regionLocation=30db5f576be8,41975,1732040928526 2024-11-19T18:29:04,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, REOPEN/MOVE because future has completed 2024-11-19T18:29:04,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,41975,1732040928526}] 2024-11-19T18:29:04,831 INFO [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,831 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 74fc47bc7ee81b8f5f6b4b68f2317277, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:04,831 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,831 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:04,831 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,831 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,833 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,834 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74fc47bc7ee81b8f5f6b4b68f2317277 columnFamilyName cf1 2024-11-19T18:29:04,834 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:04,841 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/09bd95c77fec4137914dad15ca93e0c6 2024-11-19T18:29:04,841 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(327): Store=74fc47bc7ee81b8f5f6b4b68f2317277/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:04,841 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,842 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74fc47bc7ee81b8f5f6b4b68f2317277 columnFamilyName cf2 2024-11-19T18:29:04,842 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:04,848 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf2/d347fe4fb1cb4231a911dcab472bde79 2024-11-19T18:29:04,848 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(327): Store=74fc47bc7ee81b8f5f6b4b68f2317277/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:04,849 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,849 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,851 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,851 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,851 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,852 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-19T18:29:04,854 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,854 INFO [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 74fc47bc7ee81b8f5f6b4b68f2317277; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68393746, jitterRate=0.019146233797073364}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-19T18:29:04,855 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,855 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 74fc47bc7ee81b8f5f6b4b68f2317277: Running coprocessor pre-open hook at 1732040944832Writing region info on filesystem at 1732040944832Initializing all the Stores at 1732040944832Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040944832Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040944833 (+1 ms)Cleaning up temporary data from old regions at 1732040944851 (+18 ms)Running coprocessor post-open hooks at 1732040944855 (+4 ms)Region opened successfully at 1732040944855 2024-11-19T18:29:04,856 INFO [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., pid=12, masterSystemTime=1732040944826 2024-11-19T18:29:04,859 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,859 INFO [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,860 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=74fc47bc7ee81b8f5f6b4b68f2317277, regionState=OPEN, openSeqNum=18, regionLocation=30db5f576be8,41975,1732040928526 2024-11-19T18:29:04,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,41975,1732040928526 because future has completed 2024-11-19T18:29:04,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-19T18:29:04,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,41975,1732040928526 in 189 msec 2024-11-19T18:29:04,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, REOPEN/MOVE in 563 msec 2024-11-19T18:29:04,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T18:29:04,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T18:29:04,911 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 30db5f576be8,41975,1732040928526: testing ***** 2024-11-19T18:29:04,911 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-19T18:29:04,914 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-19T18:29:04,915 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-19T18:29:04,917 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-19T18:29:04,917 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-19T18:29:04,924 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 239114976 }, "NonHeapMemoryUsage": { "committed": 170917888, "init": 7667712, "max": -1, "used": 168203104 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "30db5f576be8", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 12, "ProcessCallTime_mean": 5, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 6, "ProcessCallTime_75th_percentile": 9, "ProcessCallTime_90th_percentile": 10, "ProcessCallTime_95th_percentile": 11, "ProcessCallTime_98th_percentile": 11, "ProcessCallTime_99th_percentile": 11, "ProcessCallTime_99.9th_percentile": 11, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 12, "TotalCallTime_mean": 5, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 6, "TotalCallTime_75th_percentile": 9, "TotalCallTime_90th_percentile": 10, "TotalCallTime_95th_percentile": 11, "TotalCallTime_98th_percentile": 11, "TotalCallTime_99th_percentile": 11, "TotalCallTime_99.9th_percentile": 11, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "30db5f576be8", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:52143", "tag.serverName": "30db5f576be8,38055,1732040928383", "tag.clusterId": "f2f4b32a-28d6-4df3-bcd2-a308881871fc", "tag.Context": "regionserver", "tag.Hostname": "30db5f576be8", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732040928383, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-19T18:29:04,928 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42887 {}] master.MasterRpcServices(700): 30db5f576be8,41975,1732040928526 reported a fatal error: ***** ABORTING region server 30db5f576be8,41975,1732040928526: testing ***** 2024-11-19T18:29:04,930 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '30db5f576be8,41975,1732040928526' ***** 2024-11-19T18:29:04,930 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-19T18:29:04,930 INFO [RS:2;30db5f576be8:41975 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T18:29:04,931 INFO [RS:2;30db5f576be8:41975 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-19T18:29:04,931 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T18:29:04,931 INFO [RS:2;30db5f576be8:41975 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-19T18:29:04,931 INFO [RS:2;30db5f576be8:41975 {}] regionserver.HRegionServer(3091): Received CLOSE for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38055 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:55162 deadline: 1732041004931, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=30db5f576be8 port=41975 startCode=1732040928526. As of locationSeqNum=12. 2024-11-19T18:29:04,932 INFO [RS:2;30db5f576be8:41975 {}] regionserver.HRegionServer(956): aborting server 30db5f576be8,41975,1732040928526 2024-11-19T18:29:04,932 INFO [RS:2;30db5f576be8:41975 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T18:29:04,932 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 74fc47bc7ee81b8f5f6b4b68f2317277, disabling compactions & flushes 2024-11-19T18:29:04,932 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,932 INFO [RS:2;30db5f576be8:41975 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;30db5f576be8:41975. 2024-11-19T18:29:04,932 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,932 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. after waiting 0 ms 2024-11-19T18:29:04,932 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,932 DEBUG [RS:2;30db5f576be8:41975 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T18:29:04,932 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,38055,1732040928383, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,38055,1732040928383, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=30db5f576be8 port=41975 startCode=1732040928526. As of locationSeqNum=12. 2024-11-19T18:29:04,932 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,38055,1732040928383, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=30db5f576be8 port=41975 startCode=1732040928526. As of locationSeqNum=12. 2024-11-19T18:29:04,932 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,38055,1732040928383, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=30db5f576be8 port=41975 startCode=1732040928526. As of locationSeqNum=12. 2024-11-19T18:29:04,933 DEBUG [RS:2;30db5f576be8:41975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T18:29:04,933 INFO [RS:2;30db5f576be8:41975 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T18:29:04,933 DEBUG [RS:2;30db5f576be8:41975 {}] regionserver.HRegionServer(1325): Online Regions={74fc47bc7ee81b8f5f6b4b68f2317277=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.} 2024-11-19T18:29:04,934 DEBUG [RS:2;30db5f576be8:41975 {}] regionserver.HRegionServer(1351): Waiting on 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:04,936 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,936 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 74fc47bc7ee81b8f5f6b4b68f2317277: Waiting for close lock at 1732040944932Running coprocessor pre-close hooks at 1732040944932Disabling compacts and flushes for region at 1732040944932Disabling writes for close at 1732040944932Writing region close event to WAL at 1732040944935 (+3 ms)Running coprocessor post-close hooks at 1732040944935Closed at 1732040944936 (+1 ms) 2024-11-19T18:29:04,936 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:04,987 INFO [regionserver/30db5f576be8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T18:29:05,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 30db5f576be8,41975,1732040928526 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:05,041 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 30db5f576be8,41975,1732040928526 aborting 2024-11-19T18:29:05,041 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 30db5f576be8,41975,1732040928526 aborting 2024-11-19T18:29:05,041 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=12 from cache 2024-11-19T18:29:05,134 INFO [RS:2;30db5f576be8:41975 {}] regionserver.HRegionServer(976): stopping server 30db5f576be8,41975,1732040928526; all regions closed. 2024-11-19T18:29:05,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741832_1008 (size=2142) 2024-11-19T18:29:05,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741832_1008 (size=2142) 2024-11-19T18:29:05,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741832_1008 (size=2142) 2024-11-19T18:29:05,139 DEBUG [RS:2;30db5f576be8:41975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T18:29:05,139 INFO [RS:2;30db5f576be8:41975 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T18:29:05,139 INFO [RS:2;30db5f576be8:41975 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T18:29:05,139 INFO [RS:2;30db5f576be8:41975 {}] hbase.ChoreService(370): Chore service for: regionserver/30db5f576be8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T18:29:05,140 INFO [regionserver/30db5f576be8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T18:29:05,140 INFO [RS:2;30db5f576be8:41975 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T18:29:05,140 INFO [RS:2;30db5f576be8:41975 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T18:29:05,140 INFO [RS:2;30db5f576be8:41975 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T18:29:05,140 INFO [RS:2;30db5f576be8:41975 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T18:29:05,140 INFO [RS:2;30db5f576be8:41975 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41975 2024-11-19T18:29:05,148 INFO [RS:2;30db5f576be8:41975 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T18:29:05,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T18:29:05,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41975-0x101317d11230003, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/30db5f576be8,41975,1732040928526 2024-11-19T18:29:05,149 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [30db5f576be8,41975,1732040928526] 2024-11-19T18:29:05,152 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/30db5f576be8,41975,1732040928526 already deleted, retry=false 2024-11-19T18:29:05,152 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 30db5f576be8,41975,1732040928526 on 30db5f576be8,42887,1732040927686 2024-11-19T18:29:05,157 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 30db5f576be8,41975,1732040928526, splitWal=true, meta=false 2024-11-19T18:29:05,159 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for 30db5f576be8,41975,1732040928526 (carryingMeta=false) 30db5f576be8,41975,1732040928526/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3c39ccf2[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-19T18:29:05,161 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 30db5f576be8,41975,1732040928526, splitWal=true, meta=false 2024-11-19T18:29:05,163 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 30db5f576be8,41975,1732040928526 had 1 regions 2024-11-19T18:29:05,164 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 30db5f576be8,41975,1732040928526, splitWal=true, meta=false, isMeta: false 2024-11-19T18:29:05,166 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526-splitting 2024-11-19T18:29:05,167 INFO [PEWorker-1 {}] master.SplitWALManager(105): 30db5f576be8,41975,1732040928526 WAL count=1, meta=false 2024-11-19T18:29:05,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 30db5f576be8%2C41975%2C1732040928526.1732040929838}] 2024-11-19T18:29:05,175 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=30db5f576be8,40263,1732040928486 2024-11-19T18:29:05,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 30db5f576be8%2C41975%2C1732040928526.1732040929838, worker=30db5f576be8,40263,1732040928486}] 2024-11-19T18:29:05,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41975-0x101317d11230003, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T18:29:05,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41975-0x101317d11230003, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T18:29:05,252 INFO [RS:2;30db5f576be8:41975 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T18:29:05,252 INFO [RS:2;30db5f576be8:41975 {}] regionserver.HRegionServer(1031): Exiting; stopping=30db5f576be8,41975,1732040928526; zookeeper connection closed. 2024-11-19T18:29:05,252 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4085bcd8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4085bcd8 2024-11-19T18:29:05,253 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=18] 2024-11-19T18:29:05,255 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 30db5f576be8:41975 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 30db5f576be8/172.17.0.2:41975 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:05,256 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=18, error=java.net.ConnectException: Call to address=30db5f576be8:41975 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 30db5f576be8/172.17.0.2:41975 2024-11-19T18:29:05,256 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-19T18:29:05,256 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=18 from cache 2024-11-19T18:29:05,256 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 30db5f576be8:41975 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 30db5f576be8/172.17.0.2:41975 2024-11-19T18:29:05,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40263 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-19T18:29:05,360 INFO [RS_LOG_REPLAY_OPS-regionserver/30db5f576be8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526-splitting/30db5f576be8%2C41975%2C1732040928526.1732040929838, size=2.1 K (2142bytes) 2024-11-19T18:29:05,360 INFO [RS_LOG_REPLAY_OPS-regionserver/30db5f576be8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526-splitting/30db5f576be8%2C41975%2C1732040928526.1732040929838 2024-11-19T18:29:05,361 INFO [RS_LOG_REPLAY_OPS-regionserver/30db5f576be8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526-splitting/30db5f576be8%2C41975%2C1732040928526.1732040929838 after 1ms 2024-11-19T18:29:05,364 INFO [RS_LOG_REPLAY_OPS-regionserver/30db5f576be8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526-splitting/30db5f576be8%2C41975%2C1732040928526.1732040929838 took 4ms 2024-11-19T18:29:05,375 DEBUG [RS_LOG_REPLAY_OPS-regionserver/30db5f576be8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 74fc47bc7ee81b8f5f6b4b68f2317277: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-19T18:29:05,375 DEBUG [RS_LOG_REPLAY_OPS-regionserver/30db5f576be8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526-splitting/30db5f576be8%2C41975%2C1732040928526.1732040929838 so closing down 2024-11-19T18:29:05,375 DEBUG [RS_LOG_REPLAY_OPS-regionserver/30db5f576be8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T18:29:05,375 INFO [RS_LOG_REPLAY_OPS-regionserver/30db5f576be8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T18:29:05,375 INFO [RS_LOG_REPLAY_OPS-regionserver/30db5f576be8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 9 ms; skipped=6; WAL=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526-splitting/30db5f576be8%2C41975%2C1732040928526.1732040929838, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-11-19T18:29:05,376 DEBUG [RS_LOG_REPLAY_OPS-regionserver/30db5f576be8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526-splitting/30db5f576be8%2C41975%2C1732040928526.1732040929838, journal: Splitting hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526-splitting/30db5f576be8%2C41975%2C1732040928526.1732040929838, size=2.1 K (2142bytes) at 1732040945360Finishing writing output for hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526-splitting/30db5f576be8%2C41975%2C1732040928526.1732040929838 so closing down at 1732040945375 (+15 ms)3 split writer threads finished at 1732040945375Processed 6 edits across 0 Regions in 9 ms; skipped=6; WAL=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526-splitting/30db5f576be8%2C41975%2C1732040928526.1732040929838, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1732040945375 2024-11-19T18:29:05,376 DEBUG [RS_LOG_REPLAY_OPS-regionserver/30db5f576be8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526-splitting/30db5f576be8%2C41975%2C1732040928526.1732040929838 2024-11-19T18:29:05,377 DEBUG [RS_LOG_REPLAY_OPS-regionserver/30db5f576be8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-19T18:29:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42887 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-19T18:29:05,385 INFO [PEWorker-3 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526-splitting/30db5f576be8%2C41975%2C1732040928526.1732040929838 to hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/oldWALs 2024-11-19T18:29:05,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-19T18:29:05,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 30db5f576be8%2C41975%2C1732040928526.1732040929838, worker=30db5f576be8,40263,1732040928486 in 208 msec 2024-11-19T18:29:05,389 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=30db5f576be8,40263,1732040928486 2024-11-19T18:29:05,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-19T18:29:05,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 30db5f576be8%2C41975%2C1732040928526.1732040929838, worker=30db5f576be8,40263,1732040928486 in 220 msec 2024-11-19T18:29:05,393 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526-splitting dir is empty, no logs to split. 2024-11-19T18:29:05,393 INFO [PEWorker-1 {}] master.SplitWALManager(105): 30db5f576be8,41975,1732040928526 WAL count=0, meta=false 2024-11-19T18:29:05,393 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 30db5f576be8,41975,1732040928526 WAL splitting is done? wals=0, meta=false 2024-11-19T18:29:05,395 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 30db5f576be8,41975,1732040928526 failed, ignore...File hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,41975,1732040928526-splitting does not exist. 2024-11-19T18:29:05,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, ASSIGN}] 2024-11-19T18:29:05,399 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, ASSIGN 2024-11-19T18:29:05,399 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T18:29:05,550 DEBUG [30db5f576be8:42887 {}] balancer.BalancerClusterState(204): Hosts are {30db5f576be8=0} racks are {/default-rack=0} 2024-11-19T18:29:05,550 DEBUG [30db5f576be8:42887 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T18:29:05,550 DEBUG [30db5f576be8:42887 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T18:29:05,550 DEBUG [30db5f576be8:42887 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T18:29:05,550 DEBUG [30db5f576be8:42887 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T18:29:05,550 INFO [30db5f576be8:42887 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T18:29:05,550 INFO [30db5f576be8:42887 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T18:29:05,550 DEBUG [30db5f576be8:42887 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T18:29:05,551 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=74fc47bc7ee81b8f5f6b4b68f2317277, regionState=OPENING, regionLocation=30db5f576be8,40263,1732040928486 2024-11-19T18:29:05,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, ASSIGN because future has completed 2024-11-19T18:29:05,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,40263,1732040928486}] 2024-11-19T18:29:05,573 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=18] 2024-11-19T18:29:05,573 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 30db5f576be8:41975 this server is in the failed servers list 2024-11-19T18:29:05,574 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=30db5f576be8:41975 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 30db5f576be8:41975 2024-11-19T18:29:05,574 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 30db5f576be8:41975 2024-11-19T18:29:05,574 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,41975,1732040928526, seqNum=18 from cache 2024-11-19T18:29:05,712 INFO [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:05,712 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 74fc47bc7ee81b8f5f6b4b68f2317277, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:05,713 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:05,713 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:05,713 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:05,713 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:05,714 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:05,715 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74fc47bc7ee81b8f5f6b4b68f2317277 columnFamilyName cf1 2024-11-19T18:29:05,715 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:05,722 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf1/09bd95c77fec4137914dad15ca93e0c6 2024-11-19T18:29:05,722 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(327): Store=74fc47bc7ee81b8f5f6b4b68f2317277/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:05,722 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:05,723 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74fc47bc7ee81b8f5f6b4b68f2317277 columnFamilyName cf2 2024-11-19T18:29:05,723 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:05,729 DEBUG [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/cf2/d347fe4fb1cb4231a911dcab472bde79 2024-11-19T18:29:05,730 INFO [StoreOpener-74fc47bc7ee81b8f5f6b4b68f2317277-1 {}] regionserver.HStore(327): Store=74fc47bc7ee81b8f5f6b4b68f2317277/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:05,730 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:05,730 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:05,732 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:05,732 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:05,732 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:05,733 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-19T18:29:05,734 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:05,735 INFO [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 74fc47bc7ee81b8f5f6b4b68f2317277; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59854840, jitterRate=-0.10809338092803955}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-19T18:29:05,735 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:05,736 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 74fc47bc7ee81b8f5f6b4b68f2317277: Running coprocessor pre-open hook at 1732040945713Writing region info on filesystem at 1732040945713Initializing all the Stores at 1732040945714 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040945714Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040945714Cleaning up temporary data from old regions at 1732040945732 (+18 ms)Running coprocessor post-open hooks at 1732040945735 (+3 ms)Region opened successfully at 1732040945736 (+1 ms) 2024-11-19T18:29:05,737 INFO [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., pid=17, masterSystemTime=1732040945708 2024-11-19T18:29:05,740 DEBUG [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:05,740 INFO [RS_OPEN_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:05,740 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=74fc47bc7ee81b8f5f6b4b68f2317277, regionState=OPEN, openSeqNum=18, regionLocation=30db5f576be8,40263,1732040928486 2024-11-19T18:29:05,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,40263,1732040928486 because future has completed 2024-11-19T18:29:05,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-19T18:29:05,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 74fc47bc7ee81b8f5f6b4b68f2317277, server=30db5f576be8,40263,1732040928486 in 189 msec 2024-11-19T18:29:05,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-19T18:29:05,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=74fc47bc7ee81b8f5f6b4b68f2317277, ASSIGN in 349 msec 2024-11-19T18:29:05,749 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(291): removed crashed server 30db5f576be8,41975,1732040928526 after splitting done 2024-11-19T18:29:05,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 30db5f576be8,41975,1732040928526, splitWal=true, meta=false in 596 msec 2024-11-19T18:29:06,093 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277., hostname=30db5f576be8,40263,1732040928486, seqNum=18] 2024-11-19T18:29:06,111 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=409 (was 412), OpenFileDescriptor=1053 (was 1025) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=177 (was 177), ProcessCount=11 (was 11), AvailableMemoryMB=7068 (was 7176) 2024-11-19T18:29:06,111 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1053 is superior to 1024 2024-11-19T18:29:06,126 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=409, OpenFileDescriptor=1053, MaxFileDescriptor=1048576, SystemLoadAverage=177, ProcessCount=11, AvailableMemoryMB=7066 2024-11-19T18:29:06,126 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1053 is superior to 1024 2024-11-19T18:29:06,148 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:29:06,150 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:29:06,151 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T18:29:06,154 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-97747171, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/hregion-97747171, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:29:06,168 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-97747171/hregion-97747171.1732040946154, exclude list is [], retry=0 2024-11-19T18:29:06,171 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:29:06,171 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:29:06,172 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:29:06,176 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-97747171/hregion-97747171.1732040946154 2024-11-19T18:29:06,177 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:33801:33801)] 2024-11-19T18:29:06,177 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b100cc52c48307d030436b839b36ba73, NAME => 'testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42267/hbase 2024-11-19T18:29:06,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741893_1071 (size=67) 2024-11-19T18:29:06,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741893_1071 (size=67) 2024-11-19T18:29:06,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741893_1071 (size=67) 2024-11-19T18:29:06,187 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:06,188 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,189 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b100cc52c48307d030436b839b36ba73 columnFamilyName a 2024-11-19T18:29:06,190 DEBUG [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:06,190 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(327): Store=b100cc52c48307d030436b839b36ba73/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:06,190 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,192 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b100cc52c48307d030436b839b36ba73 columnFamilyName b 2024-11-19T18:29:06,192 DEBUG [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:06,192 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(327): Store=b100cc52c48307d030436b839b36ba73/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:06,193 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,194 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b100cc52c48307d030436b839b36ba73 columnFamilyName c 2024-11-19T18:29:06,194 DEBUG [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:06,194 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(327): Store=b100cc52c48307d030436b839b36ba73/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:06,195 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,195 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,196 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,197 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,197 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,197 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:29:06,198 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,201 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T18:29:06,201 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b100cc52c48307d030436b839b36ba73; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68946621, jitterRate=0.027384713292121887}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:29:06,202 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b100cc52c48307d030436b839b36ba73: Writing region info on filesystem at 1732040946187Initializing all the Stores at 1732040946187Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040946187Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040946188 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040946188Cleaning up temporary data from old regions at 1732040946197 (+9 ms)Region opened successfully at 1732040946201 (+4 ms) 2024-11-19T18:29:06,202 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b100cc52c48307d030436b839b36ba73, disabling compactions & flushes 2024-11-19T18:29:06,202 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73. 2024-11-19T18:29:06,202 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73. 2024-11-19T18:29:06,202 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73. after waiting 0 ms 2024-11-19T18:29:06,202 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73. 2024-11-19T18:29:06,202 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73. 2024-11-19T18:29:06,202 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b100cc52c48307d030436b839b36ba73: Waiting for close lock at 1732040946202Disabling compacts and flushes for region at 1732040946202Disabling writes for close at 1732040946202Writing region close event to WAL at 1732040946202Closed at 1732040946202 2024-11-19T18:29:06,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741892_1070 (size=93) 2024-11-19T18:29:06,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741892_1070 (size=93) 2024-11-19T18:29:06,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741892_1070 (size=93) 2024-11-19T18:29:06,208 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-19T18:29:06,208 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-97747171:(num 1732040946154) 2024-11-19T18:29:06,208 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:29:06,210 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:29:06,223 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946210, exclude list is [], retry=0 2024-11-19T18:29:06,226 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:29:06,226 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:29:06,227 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:29:06,229 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946210 2024-11-19T18:29:06,229 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:37769:37769)] 2024-11-19T18:29:06,229 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b100cc52c48307d030436b839b36ba73, NAME => 'testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:06,229 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:06,230 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,230 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,231 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,232 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b100cc52c48307d030436b839b36ba73 columnFamilyName a 2024-11-19T18:29:06,232 DEBUG [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:06,233 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(327): Store=b100cc52c48307d030436b839b36ba73/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:06,233 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,234 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b100cc52c48307d030436b839b36ba73 columnFamilyName b 2024-11-19T18:29:06,234 DEBUG [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:06,234 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(327): Store=b100cc52c48307d030436b839b36ba73/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:06,234 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,235 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b100cc52c48307d030436b839b36ba73 columnFamilyName c 2024-11-19T18:29:06,235 DEBUG [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:06,236 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(327): Store=b100cc52c48307d030436b839b36ba73/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:06,236 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,237 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,238 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,239 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,239 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,240 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:29:06,241 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,242 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b100cc52c48307d030436b839b36ba73; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67274155, jitterRate=0.002463027834892273}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:29:06,242 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b100cc52c48307d030436b839b36ba73: Writing region info on filesystem at 1732040946230Initializing all the Stores at 1732040946231 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040946231Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040946231Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040946231Cleaning up temporary data from old regions at 1732040946239 (+8 ms)Region opened successfully at 1732040946242 (+3 ms) 2024-11-19T18:29:06,269 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b100cc52c48307d030436b839b36ba73 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-19T18:29:06,293 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/.tmp/a/16985ac86e7d4d36bc6cd37b63c7016d is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732040946242/Put/seqid=0 2024-11-19T18:29:06,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741895_1073 (size=5958) 2024-11-19T18:29:06,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741895_1073 (size=5958) 2024-11-19T18:29:06,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741895_1073 (size=5958) 2024-11-19T18:29:06,302 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/.tmp/a/16985ac86e7d4d36bc6cd37b63c7016d 2024-11-19T18:29:06,331 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/.tmp/b/2176d462879045528c4ac7fefa35341f is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732040946250/Put/seqid=0 2024-11-19T18:29:06,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741896_1074 (size=5958) 2024-11-19T18:29:06,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741896_1074 (size=5958) 2024-11-19T18:29:06,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741896_1074 (size=5958) 2024-11-19T18:29:06,344 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/.tmp/b/2176d462879045528c4ac7fefa35341f 2024-11-19T18:29:06,365 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/.tmp/c/49e37122ac044bc6bb3f7e6ec55a68f4 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732040946258/Put/seqid=0 2024-11-19T18:29:06,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741897_1075 (size=5958) 2024-11-19T18:29:06,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741897_1075 (size=5958) 2024-11-19T18:29:06,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741897_1075 (size=5958) 2024-11-19T18:29:06,374 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/.tmp/c/49e37122ac044bc6bb3f7e6ec55a68f4 2024-11-19T18:29:06,384 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/.tmp/a/16985ac86e7d4d36bc6cd37b63c7016d as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/a/16985ac86e7d4d36bc6cd37b63c7016d 2024-11-19T18:29:06,390 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/a/16985ac86e7d4d36bc6cd37b63c7016d, entries=10, sequenceid=33, filesize=5.8 K 2024-11-19T18:29:06,391 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/.tmp/b/2176d462879045528c4ac7fefa35341f as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/b/2176d462879045528c4ac7fefa35341f 2024-11-19T18:29:06,397 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/b/2176d462879045528c4ac7fefa35341f, entries=10, sequenceid=33, filesize=5.8 K 2024-11-19T18:29:06,398 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/.tmp/c/49e37122ac044bc6bb3f7e6ec55a68f4 as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/c/49e37122ac044bc6bb3f7e6ec55a68f4 2024-11-19T18:29:06,403 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/c/49e37122ac044bc6bb3f7e6ec55a68f4, entries=10, sequenceid=33, filesize=5.8 K 2024-11-19T18:29:06,405 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for b100cc52c48307d030436b839b36ba73 in 136ms, sequenceid=33, compaction requested=false 2024-11-19T18:29:06,405 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b100cc52c48307d030436b839b36ba73: 2024-11-19T18:29:06,406 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b100cc52c48307d030436b839b36ba73, disabling compactions & flushes 2024-11-19T18:29:06,406 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73. 2024-11-19T18:29:06,406 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73. 2024-11-19T18:29:06,406 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73. after waiting 0 ms 2024-11-19T18:29:06,406 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73. 2024-11-19T18:29:06,412 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73. 2024-11-19T18:29:06,412 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b100cc52c48307d030436b839b36ba73: Waiting for close lock at 1732040946405Disabling compacts and flushes for region at 1732040946405Disabling writes for close at 1732040946406 (+1 ms)Writing region close event to WAL at 1732040946411 (+5 ms)Closed at 1732040946411 2024-11-19T18:29:06,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741894_1072 (size=5937) 2024-11-19T18:29:06,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741894_1072 (size=5937) 2024-11-19T18:29:06,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741894_1072 (size=5937) 2024-11-19T18:29:06,420 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/b/2176d462879045528c4ac7fefa35341f to hdfs://localhost:42267/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/b/2176d462879045528c4ac7fefa35341f 2024-11-19T18:29:06,445 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42267/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946210, size=5.8 K (5937bytes) 2024-11-19T18:29:06,445 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42267/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946210 2024-11-19T18:29:06,446 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42267/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946210 after 1ms 2024-11-19T18:29:06,449 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42267/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946210 took 4ms 2024-11-19T18:29:06,451 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42267/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946210 so closing down 2024-11-19T18:29:06,452 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T18:29:06,453 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732040946210.temp 2024-11-19T18:29:06,455 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/recovered.edits/0000000000000000003-wal.1732040946210.temp 2024-11-19T18:29:06,455 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T18:29:06,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741898_1076 (size=5223) 2024-11-19T18:29:06,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741898_1076 (size=5223) 2024-11-19T18:29:06,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741898_1076 (size=5223) 2024-11-19T18:29:06,465 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/recovered.edits/0000000000000000003-wal.1732040946210.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-19T18:29:06,466 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/recovered.edits/0000000000000000003-wal.1732040946210.temp to hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/recovered.edits/0000000000000000032 2024-11-19T18:29:06,467 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 18 ms; skipped=2; WAL=hdfs://localhost:42267/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946210, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-11-19T18:29:06,467 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42267/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946210, journal: Splitting hdfs://localhost:42267/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946210, size=5.8 K (5937bytes) at 1732040946445Finishing writing output for hdfs://localhost:42267/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946210 so closing down at 1732040946451 (+6 ms)Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/recovered.edits/0000000000000000003-wal.1732040946210.temp at 1732040946455 (+4 ms)3 split writer threads finished at 1732040946455Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/recovered.edits/0000000000000000003-wal.1732040946210.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732040946465 (+10 ms)Rename recovered edits hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/recovered.edits/0000000000000000003-wal.1732040946210.temp to hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/recovered.edits/0000000000000000032 at 1732040946466 (+1 ms)Processed 32 edits across 1 Regions in 18 ms; skipped=2; WAL=hdfs://localhost:42267/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946210, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1732040946467 (+1 ms) 2024-11-19T18:29:06,469 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42267/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946210 to hdfs://localhost:42267/hbase/oldWALs/wal.1732040946210 2024-11-19T18:29:06,470 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/recovered.edits/0000000000000000032 2024-11-19T18:29:06,470 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:29:06,472 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:29:06,493 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946473, exclude list is [], retry=0 2024-11-19T18:29:06,497 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:29:06,497 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:29:06,498 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:29:06,505 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946473 2024-11-19T18:29:06,505 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:33801:33801)] 2024-11-19T18:29:06,505 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b100cc52c48307d030436b839b36ba73, NAME => 'testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:06,506 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:06,506 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,506 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,508 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,510 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b100cc52c48307d030436b839b36ba73 columnFamilyName a 2024-11-19T18:29:06,510 DEBUG [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:06,517 DEBUG [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/a/16985ac86e7d4d36bc6cd37b63c7016d 2024-11-19T18:29:06,517 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(327): Store=b100cc52c48307d030436b839b36ba73/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:06,517 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,519 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b100cc52c48307d030436b839b36ba73 columnFamilyName b 2024-11-19T18:29:06,519 DEBUG [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:06,519 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(327): Store=b100cc52c48307d030436b839b36ba73/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:06,520 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,521 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b100cc52c48307d030436b839b36ba73 columnFamilyName c 2024-11-19T18:29:06,521 DEBUG [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:06,527 DEBUG [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/c/49e37122ac044bc6bb3f7e6ec55a68f4 2024-11-19T18:29:06,528 INFO [StoreOpener-b100cc52c48307d030436b839b36ba73-1 {}] regionserver.HStore(327): Store=b100cc52c48307d030436b839b36ba73/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:06,528 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,529 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,530 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,531 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/recovered.edits/0000000000000000032 2024-11-19T18:29:06,535 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/recovered.edits/0000000000000000032 2024-11-19T18:29:06,535 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b100cc52c48307d030436b839b36ba73 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-19T18:29:06,558 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/.tmp/b/9e1b50331f054db4a08c219eb98d4138 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732040946250/Put/seqid=0 2024-11-19T18:29:06,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741900_1078 (size=5958) 2024-11-19T18:29:06,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741900_1078 (size=5958) 2024-11-19T18:29:06,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741900_1078 (size=5958) 2024-11-19T18:29:06,567 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/.tmp/b/9e1b50331f054db4a08c219eb98d4138 2024-11-19T18:29:06,574 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/.tmp/b/9e1b50331f054db4a08c219eb98d4138 as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/b/9e1b50331f054db4a08c219eb98d4138 2024-11-19T18:29:06,582 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/b/9e1b50331f054db4a08c219eb98d4138, entries=10, sequenceid=32, filesize=5.8 K 2024-11-19T18:29:06,583 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for b100cc52c48307d030436b839b36ba73 in 47ms, sequenceid=32, compaction requested=false; wal=null 2024-11-19T18:29:06,584 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/recovered.edits/0000000000000000032 2024-11-19T18:29:06,585 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,585 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,586 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:29:06,588 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b100cc52c48307d030436b839b36ba73 2024-11-19T18:29:06,591 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b100cc52c48307d030436b839b36ba73/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-19T18:29:06,592 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b100cc52c48307d030436b839b36ba73; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72146009, jitterRate=0.07505930960178375}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:29:06,593 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b100cc52c48307d030436b839b36ba73: Writing region info on filesystem at 1732040946506Initializing all the Stores at 1732040946507 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040946507Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040946508 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040946508Obtaining lock to block concurrent updates at 1732040946535 (+27 ms)Preparing flush snapshotting stores in b100cc52c48307d030436b839b36ba73 at 1732040946535Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732040946535Flushing stores of testReplayEditsWrittenViaHRegion,,1732040946149.b100cc52c48307d030436b839b36ba73. at 1732040946535Flushing b100cc52c48307d030436b839b36ba73/b: creating writer at 1732040946535Flushing b100cc52c48307d030436b839b36ba73/b: appending metadata at 1732040946557 (+22 ms)Flushing b100cc52c48307d030436b839b36ba73/b: closing flushed file at 1732040946557Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fd8f683: reopening flushed file at 1732040946573 (+16 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for b100cc52c48307d030436b839b36ba73 in 47ms, sequenceid=32, compaction requested=false; wal=null at 1732040946583 (+10 ms)Cleaning up temporary data from old regions at 1732040946585 (+2 ms)Region opened successfully at 1732040946593 (+8 ms) 2024-11-19T18:29:06,619 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=418 (was 409) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:51402 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:55702 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:52586 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:52714 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:51230 [Waiting for operation #38] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:55538 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1034186995-172.17.0.2-1732040924630:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1034186995-172.17.0.2-1732040924630:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1034186995-172.17.0.2-1732040924630:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1129 (was 1053) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=177 (was 177), ProcessCount=11 (was 11), AvailableMemoryMB=7061 (was 7066) 2024-11-19T18:29:06,620 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1129 is superior to 1024 2024-11-19T18:29:06,632 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=418, OpenFileDescriptor=1129, MaxFileDescriptor=1048576, SystemLoadAverage=177, ProcessCount=11, AvailableMemoryMB=7057 2024-11-19T18:29:06,632 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1129 is superior to 1024 2024-11-19T18:29:06,648 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:29:06,650 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:29:06,651 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T18:29:06,656 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-75388720, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/hregion-75388720, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:29:06,671 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-75388720/hregion-75388720.1732040946656, exclude list is [], retry=0 2024-11-19T18:29:06,674 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:29:06,675 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:29:06,682 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:29:06,689 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-75388720/hregion-75388720.1732040946656 2024-11-19T18:29:06,691 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:33801:33801)] 2024-11-19T18:29:06,691 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => edaf4aceed24af34ce087114ceeed32f, NAME => 'testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42267/hbase 2024-11-19T18:29:06,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741902_1080 (size=68) 2024-11-19T18:29:06,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741902_1080 (size=68) 2024-11-19T18:29:06,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741902_1080 (size=68) 2024-11-19T18:29:06,701 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:06,706 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,708 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edaf4aceed24af34ce087114ceeed32f columnFamilyName a 2024-11-19T18:29:06,708 DEBUG [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:06,708 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(327): Store=edaf4aceed24af34ce087114ceeed32f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:06,708 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,709 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edaf4aceed24af34ce087114ceeed32f columnFamilyName b 2024-11-19T18:29:06,710 DEBUG [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:06,710 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(327): Store=edaf4aceed24af34ce087114ceeed32f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:06,710 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,711 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edaf4aceed24af34ce087114ceeed32f columnFamilyName c 2024-11-19T18:29:06,711 DEBUG [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:06,712 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(327): Store=edaf4aceed24af34ce087114ceeed32f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:06,712 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,712 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,713 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,714 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,714 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,714 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:29:06,715 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,717 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T18:29:06,717 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened edaf4aceed24af34ce087114ceeed32f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74876312, jitterRate=0.11574399471282959}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:29:06,718 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for edaf4aceed24af34ce087114ceeed32f: Writing region info on filesystem at 1732040946701Initializing all the Stores at 1732040946702 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040946702Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040946706 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040946706Cleaning up temporary data from old regions at 1732040946714 (+8 ms)Region opened successfully at 1732040946718 (+4 ms) 2024-11-19T18:29:06,718 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing edaf4aceed24af34ce087114ceeed32f, disabling compactions & flushes 2024-11-19T18:29:06,718 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f. 2024-11-19T18:29:06,718 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f. 2024-11-19T18:29:06,718 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f. after waiting 0 ms 2024-11-19T18:29:06,718 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f. 2024-11-19T18:29:06,719 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f. 2024-11-19T18:29:06,719 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for edaf4aceed24af34ce087114ceeed32f: Waiting for close lock at 1732040946718Disabling compacts and flushes for region at 1732040946718Disabling writes for close at 1732040946718Writing region close event to WAL at 1732040946719 (+1 ms)Closed at 1732040946719 2024-11-19T18:29:06,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741901_1079 (size=93) 2024-11-19T18:29:06,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741901_1079 (size=93) 2024-11-19T18:29:06,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741901_1079 (size=93) 2024-11-19T18:29:06,727 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-19T18:29:06,727 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-75388720:(num 1732040946656) 2024-11-19T18:29:06,727 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:29:06,729 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:29:06,742 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040946729, exclude list is [], retry=0 2024-11-19T18:29:06,745 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:29:06,746 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:29:06,746 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:29:06,749 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040946729 2024-11-19T18:29:06,750 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:40377:40377)] 2024-11-19T18:29:06,825 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => edaf4aceed24af34ce087114ceeed32f, NAME => 'testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:06,828 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,829 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:06,829 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,829 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,831 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,832 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edaf4aceed24af34ce087114ceeed32f columnFamilyName a 2024-11-19T18:29:06,833 DEBUG [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:06,833 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(327): Store=edaf4aceed24af34ce087114ceeed32f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:06,833 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,834 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edaf4aceed24af34ce087114ceeed32f columnFamilyName b 2024-11-19T18:29:06,834 DEBUG [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:06,835 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(327): Store=edaf4aceed24af34ce087114ceeed32f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:06,835 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,836 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edaf4aceed24af34ce087114ceeed32f columnFamilyName c 2024-11-19T18:29:06,836 DEBUG [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:06,837 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(327): Store=edaf4aceed24af34ce087114ceeed32f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:06,837 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,837 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,839 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,840 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,840 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,841 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:29:06,843 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,844 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened edaf4aceed24af34ce087114ceeed32f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62887267, jitterRate=-0.06290669739246368}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:29:06,844 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:06,844 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for edaf4aceed24af34ce087114ceeed32f: Running coprocessor pre-open hook at 1732040946829Writing region info on filesystem at 1732040946829Initializing all the Stores at 1732040946830 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040946830Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040946831 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040946831Cleaning up temporary data from old regions at 1732040946840 (+9 ms)Running coprocessor post-open hooks at 1732040946844 (+4 ms)Region opened successfully at 1732040946844 2024-11-19T18:29:06,857 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing edaf4aceed24af34ce087114ceeed32f 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-19T18:29:06,858 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for edaf4aceed24af34ce087114ceeed32f/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:07,859 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for edaf4aceed24af34ce087114ceeed32f/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:08,578 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T18:29:08,608 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-19T18:29:08,608 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-19T18:29:08,609 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-19T18:29:08,609 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-19T18:29:08,609 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-19T18:29:08,609 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-19T18:29:08,610 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-19T18:29:08,610 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-19T18:29:08,860 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for edaf4aceed24af34ce087114ceeed32f/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:09,861 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for edaf4aceed24af34ce087114ceeed32f/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:10,861 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for edaf4aceed24af34ce087114ceeed32f/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:11,862 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for edaf4aceed24af34ce087114ceeed32f/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:12,863 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for edaf4aceed24af34ce087114ceeed32f/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:13,863 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for edaf4aceed24af34ce087114ceeed32f/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:14,864 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for edaf4aceed24af34ce087114ceeed32f/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:15,864 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for edaf4aceed24af34ce087114ceeed32f/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:15,866 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for edaf4aceed24af34ce087114ceeed32f: 2024-11-19T18:29:15,866 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f. 2024-11-19T18:29:15,880 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for edaf4aceed24af34ce087114ceeed32f: 2024-11-19T18:29:15,880 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-19T18:29:15,880 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing edaf4aceed24af34ce087114ceeed32f, disabling compactions & flushes 2024-11-19T18:29:15,880 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f. 2024-11-19T18:29:15,880 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f. 2024-11-19T18:29:15,880 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f. after waiting 0 ms 2024-11-19T18:29:15,880 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f. 2024-11-19T18:29:15,881 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f. 2024-11-19T18:29:15,881 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f. 2024-11-19T18:29:15,881 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for edaf4aceed24af34ce087114ceeed32f: Waiting for close lock at 1732040955880Running coprocessor pre-close hooks at 1732040955880Disabling compacts and flushes for region at 1732040955880Disabling writes for close at 1732040955880Writing region close event to WAL at 1732040955881 (+1 ms)Running coprocessor post-close hooks at 1732040955881Closed at 1732040955881 2024-11-19T18:29:15,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741903_1081 (size=3941) 2024-11-19T18:29:15,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741903_1081 (size=3941) 2024-11-19T18:29:15,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741903_1081 (size=3941) 2024-11-19T18:29:15,900 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42267/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040946729, size=3.8 K (3941bytes) 2024-11-19T18:29:15,900 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42267/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040946729 2024-11-19T18:29:15,901 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42267/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040946729 after 1ms 2024-11-19T18:29:15,903 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42267/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040946729 took 3ms 2024-11-19T18:29:15,905 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42267/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040946729 so closing down 2024-11-19T18:29:15,905 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T18:29:15,906 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732040946729.temp 2024-11-19T18:29:15,907 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/recovered.edits/0000000000000000004-wal.1732040946729.temp 2024-11-19T18:29:15,907 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T18:29:15,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741904_1082 (size=2983) 2024-11-19T18:29:15,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741904_1082 (size=2983) 2024-11-19T18:29:15,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741904_1082 (size=2983) 2024-11-19T18:29:15,920 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/recovered.edits/0000000000000000004-wal.1732040946729.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-19T18:29:15,921 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/recovered.edits/0000000000000000004-wal.1732040946729.temp to hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/recovered.edits/0000000000000000026 2024-11-19T18:29:15,921 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:42267/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040946729, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-11-19T18:29:15,922 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42267/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040946729, journal: Splitting hdfs://localhost:42267/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040946729, size=3.8 K (3941bytes) at 1732040955900Finishing writing output for hdfs://localhost:42267/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040946729 so closing down at 1732040955905 (+5 ms)Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/recovered.edits/0000000000000000004-wal.1732040946729.temp at 1732040955907 (+2 ms)3 split writer threads finished at 1732040955908 (+1 ms)Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/recovered.edits/0000000000000000004-wal.1732040946729.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732040955920 (+12 ms)Rename recovered edits hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/recovered.edits/0000000000000000004-wal.1732040946729.temp to hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/recovered.edits/0000000000000000026 at 1732040955921 (+1 ms)Processed 23 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:42267/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040946729, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1732040955921 2024-11-19T18:29:15,923 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42267/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040946729 to hdfs://localhost:42267/hbase/oldWALs/wal.1732040946729 2024-11-19T18:29:15,924 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/recovered.edits/0000000000000000026 2024-11-19T18:29:15,925 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:29:15,927 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:29:15,940 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040955927, exclude list is [], retry=0 2024-11-19T18:29:15,943 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:29:15,943 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:29:15,943 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:29:15,945 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040955927 2024-11-19T18:29:15,945 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:40377:40377)] 2024-11-19T18:29:15,946 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => edaf4aceed24af34ce087114ceeed32f, NAME => 'testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:15,946 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:15,946 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:15,947 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:15,947 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:15,948 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:15,949 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edaf4aceed24af34ce087114ceeed32f columnFamilyName a 2024-11-19T18:29:15,949 DEBUG [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:15,950 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(327): Store=edaf4aceed24af34ce087114ceeed32f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:15,950 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:15,950 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edaf4aceed24af34ce087114ceeed32f columnFamilyName b 2024-11-19T18:29:15,951 DEBUG [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:15,951 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(327): Store=edaf4aceed24af34ce087114ceeed32f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:15,951 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:15,952 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edaf4aceed24af34ce087114ceeed32f columnFamilyName c 2024-11-19T18:29:15,952 DEBUG [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:15,952 INFO [StoreOpener-edaf4aceed24af34ce087114ceeed32f-1 {}] regionserver.HStore(327): Store=edaf4aceed24af34ce087114ceeed32f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:15,952 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:15,953 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:15,954 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:15,955 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/recovered.edits/0000000000000000026 2024-11-19T18:29:15,958 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/recovered.edits/0000000000000000026 2024-11-19T18:29:15,958 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing edaf4aceed24af34ce087114ceeed32f 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-19T18:29:15,973 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/.tmp/a/26c00649cec948e38f7bbb8710fe2eb9 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732040955871/Put/seqid=0 2024-11-19T18:29:15,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741906_1084 (size=5523) 2024-11-19T18:29:15,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741906_1084 (size=5523) 2024-11-19T18:29:15,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741906_1084 (size=5523) 2024-11-19T18:29:15,980 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/.tmp/a/26c00649cec948e38f7bbb8710fe2eb9 2024-11-19T18:29:16,000 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/.tmp/b/ce8bb3a03f584435b8949d829cb03fd9 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732040955866/Put/seqid=0 2024-11-19T18:29:16,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741907_1085 (size=5524) 2024-11-19T18:29:16,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741907_1085 (size=5524) 2024-11-19T18:29:16,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741907_1085 (size=5524) 2024-11-19T18:29:16,006 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/.tmp/b/ce8bb3a03f584435b8949d829cb03fd9 2024-11-19T18:29:16,026 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/.tmp/c/a4703f5f14db43e881d9324a80fc94f4 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732040955869/Put/seqid=0 2024-11-19T18:29:16,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741908_1086 (size=5457) 2024-11-19T18:29:16,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741908_1086 (size=5457) 2024-11-19T18:29:16,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741908_1086 (size=5457) 2024-11-19T18:29:16,033 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/.tmp/c/a4703f5f14db43e881d9324a80fc94f4 2024-11-19T18:29:16,039 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/.tmp/a/26c00649cec948e38f7bbb8710fe2eb9 as hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/a/26c00649cec948e38f7bbb8710fe2eb9 2024-11-19T18:29:16,045 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/a/26c00649cec948e38f7bbb8710fe2eb9, entries=7, sequenceid=26, filesize=5.4 K 2024-11-19T18:29:16,046 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/.tmp/b/ce8bb3a03f584435b8949d829cb03fd9 as hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/b/ce8bb3a03f584435b8949d829cb03fd9 2024-11-19T18:29:16,051 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/b/ce8bb3a03f584435b8949d829cb03fd9, entries=7, sequenceid=26, filesize=5.4 K 2024-11-19T18:29:16,052 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/.tmp/c/a4703f5f14db43e881d9324a80fc94f4 as hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/c/a4703f5f14db43e881d9324a80fc94f4 2024-11-19T18:29:16,057 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/c/a4703f5f14db43e881d9324a80fc94f4, entries=6, sequenceid=26, filesize=5.3 K 2024-11-19T18:29:16,058 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for edaf4aceed24af34ce087114ceeed32f in 100ms, sequenceid=26, compaction requested=false; wal=null 2024-11-19T18:29:16,059 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/recovered.edits/0000000000000000026 2024-11-19T18:29:16,060 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:16,060 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:16,060 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:29:16,062 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:16,064 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testReplayEditsAfterAbortingFlush/edaf4aceed24af34ce087114ceeed32f/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-19T18:29:16,065 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened edaf4aceed24af34ce087114ceeed32f; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69522901, jitterRate=0.035971954464912415}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:29:16,065 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for edaf4aceed24af34ce087114ceeed32f 2024-11-19T18:29:16,066 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for edaf4aceed24af34ce087114ceeed32f: Running coprocessor pre-open hook at 1732040955947Writing region info on filesystem at 1732040955947Initializing all the Stores at 1732040955947Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040955948 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040955948Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040955948Obtaining lock to block concurrent updates at 1732040955958 (+10 ms)Preparing flush snapshotting stores in edaf4aceed24af34ce087114ceeed32f at 1732040955958Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732040955958Flushing stores of testReplayEditsAfterAbortingFlush,,1732040946649.edaf4aceed24af34ce087114ceeed32f. at 1732040955958Flushing edaf4aceed24af34ce087114ceeed32f/a: creating writer at 1732040955959 (+1 ms)Flushing edaf4aceed24af34ce087114ceeed32f/a: appending metadata at 1732040955973 (+14 ms)Flushing edaf4aceed24af34ce087114ceeed32f/a: closing flushed file at 1732040955973Flushing edaf4aceed24af34ce087114ceeed32f/b: creating writer at 1732040955986 (+13 ms)Flushing edaf4aceed24af34ce087114ceeed32f/b: appending metadata at 1732040955999 (+13 ms)Flushing edaf4aceed24af34ce087114ceeed32f/b: closing flushed file at 1732040955999Flushing edaf4aceed24af34ce087114ceeed32f/c: creating writer at 1732040956011 (+12 ms)Flushing edaf4aceed24af34ce087114ceeed32f/c: appending metadata at 1732040956025 (+14 ms)Flushing edaf4aceed24af34ce087114ceeed32f/c: closing flushed file at 1732040956026 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d93bc2b: reopening flushed file at 1732040956039 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37004896: reopening flushed file at 1732040956045 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c6893da: reopening flushed file at 1732040956051 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for edaf4aceed24af34ce087114ceeed32f in 100ms, sequenceid=26, compaction requested=false; wal=null at 1732040956058 (+7 ms)Cleaning up temporary data from old regions at 1732040956060 (+2 ms)Running coprocessor post-open hooks at 1732040956065 (+5 ms)Region opened successfully at 1732040956066 (+1 ms) 2024-11-19T18:29:16,089 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415 (was 418), OpenFileDescriptor=1183 (was 1129) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=202 (was 177) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7058 (was 7057) - AvailableMemoryMB LEAK? - 2024-11-19T18:29:16,089 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-11-19T18:29:16,102 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=415, OpenFileDescriptor=1183, MaxFileDescriptor=1048576, SystemLoadAverage=202, ProcessCount=11, AvailableMemoryMB=7057 2024-11-19T18:29:16,102 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-11-19T18:29:16,117 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:29:16,118 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:29:16,119 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T18:29:16,121 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-26030535, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/hregion-26030535, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:29:16,134 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-26030535/hregion-26030535.1732040956122, exclude list is [], retry=0 2024-11-19T18:29:16,136 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:29:16,137 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:29:16,137 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:29:16,139 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-26030535/hregion-26030535.1732040956122 2024-11-19T18:29:16,139 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:40377:40377)] 2024-11-19T18:29:16,139 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e5c3960e9f9494c68d003384ba336f5e, NAME => 'testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42267/hbase 2024-11-19T18:29:16,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741910_1088 (size=61) 2024-11-19T18:29:16,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741910_1088 (size=61) 2024-11-19T18:29:16,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741910_1088 (size=61) 2024-11-19T18:29:16,148 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:16,149 INFO [StoreOpener-e5c3960e9f9494c68d003384ba336f5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,150 INFO [StoreOpener-e5c3960e9f9494c68d003384ba336f5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5c3960e9f9494c68d003384ba336f5e columnFamilyName a 2024-11-19T18:29:16,150 DEBUG [StoreOpener-e5c3960e9f9494c68d003384ba336f5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:16,151 INFO [StoreOpener-e5c3960e9f9494c68d003384ba336f5e-1 {}] regionserver.HStore(327): Store=e5c3960e9f9494c68d003384ba336f5e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:16,151 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,152 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,152 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,152 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,152 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,154 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,156 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T18:29:16,157 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e5c3960e9f9494c68d003384ba336f5e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63715520, jitterRate=-0.05056476593017578}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T18:29:16,157 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e5c3960e9f9494c68d003384ba336f5e: Writing region info on filesystem at 1732040956148Initializing all the Stores at 1732040956149 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040956149Cleaning up temporary data from old regions at 1732040956153 (+4 ms)Region opened successfully at 1732040956157 (+4 ms) 2024-11-19T18:29:16,158 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e5c3960e9f9494c68d003384ba336f5e, disabling compactions & flushes 2024-11-19T18:29:16,158 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e. 2024-11-19T18:29:16,158 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e. 2024-11-19T18:29:16,158 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e. after waiting 0 ms 2024-11-19T18:29:16,158 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e. 2024-11-19T18:29:16,160 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e. 2024-11-19T18:29:16,160 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e5c3960e9f9494c68d003384ba336f5e: Waiting for close lock at 1732040956157Disabling compacts and flushes for region at 1732040956157Disabling writes for close at 1732040956158 (+1 ms)Writing region close event to WAL at 1732040956160 (+2 ms)Closed at 1732040956160 2024-11-19T18:29:16,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741909_1087 (size=93) 2024-11-19T18:29:16,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741909_1087 (size=93) 2024-11-19T18:29:16,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741909_1087 (size=93) 2024-11-19T18:29:16,166 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-19T18:29:16,166 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-26030535:(num 1732040956122) 2024-11-19T18:29:16,166 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:29:16,167 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:29:16,181 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956168, exclude list is [], retry=0 2024-11-19T18:29:16,183 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:29:16,183 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:29:16,184 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:29:16,185 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956168 2024-11-19T18:29:16,186 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:40377:40377)] 2024-11-19T18:29:16,186 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e5c3960e9f9494c68d003384ba336f5e, NAME => 'testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:16,186 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:16,186 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,186 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,187 INFO [StoreOpener-e5c3960e9f9494c68d003384ba336f5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,188 INFO [StoreOpener-e5c3960e9f9494c68d003384ba336f5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5c3960e9f9494c68d003384ba336f5e columnFamilyName a 2024-11-19T18:29:16,188 DEBUG [StoreOpener-e5c3960e9f9494c68d003384ba336f5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:16,188 INFO [StoreOpener-e5c3960e9f9494c68d003384ba336f5e-1 {}] regionserver.HStore(327): Store=e5c3960e9f9494c68d003384ba336f5e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:16,189 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,189 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,190 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,190 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,190 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,192 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,193 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e5c3960e9f9494c68d003384ba336f5e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74385714, jitterRate=0.10843351483345032}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T18:29:16,193 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e5c3960e9f9494c68d003384ba336f5e: Writing region info on filesystem at 1732040956186Initializing all the Stores at 1732040956187 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040956187Cleaning up temporary data from old regions at 1732040956190 (+3 ms)Region opened successfully at 1732040956193 (+3 ms) 2024-11-19T18:29:16,201 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e5c3960e9f9494c68d003384ba336f5e, disabling compactions & flushes 2024-11-19T18:29:16,201 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e. 2024-11-19T18:29:16,201 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e. 2024-11-19T18:29:16,201 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e. after waiting 0 ms 2024-11-19T18:29:16,201 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e. 2024-11-19T18:29:16,202 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e. 2024-11-19T18:29:16,202 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e. 2024-11-19T18:29:16,202 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e5c3960e9f9494c68d003384ba336f5e: Waiting for close lock at 1732040956201Disabling compacts and flushes for region at 1732040956201Disabling writes for close at 1732040956201Writing region close event to WAL at 1732040956202 (+1 ms)Closed at 1732040956202 2024-11-19T18:29:16,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741911_1089 (size=1623) 2024-11-19T18:29:16,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741911_1089 (size=1623) 2024-11-19T18:29:16,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741911_1089 (size=1623) 2024-11-19T18:29:16,220 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42267/hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956168, size=1.6 K (1623bytes) 2024-11-19T18:29:16,220 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42267/hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956168 2024-11-19T18:29:16,221 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42267/hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956168 after 1ms 2024-11-19T18:29:16,223 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42267/hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956168 took 3ms 2024-11-19T18:29:16,224 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42267/hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956168 so closing down 2024-11-19T18:29:16,224 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T18:29:16,226 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732040956168.temp 2024-11-19T18:29:16,227 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/recovered.edits/0000000000000000003-wal.1732040956168.temp 2024-11-19T18:29:16,227 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T18:29:16,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741912_1090 (size=1623) 2024-11-19T18:29:16,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741912_1090 (size=1623) 2024-11-19T18:29:16,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741912_1090 (size=1623) 2024-11-19T18:29:16,234 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/recovered.edits/0000000000000000003-wal.1732040956168.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-19T18:29:16,235 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/recovered.edits/0000000000000000003-wal.1732040956168.temp to hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/recovered.edits/0000000000000000012 2024-11-19T18:29:16,235 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956168, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-11-19T18:29:16,235 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42267/hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956168, journal: Splitting hdfs://localhost:42267/hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956168, size=1.6 K (1623bytes) at 1732040956220Finishing writing output for hdfs://localhost:42267/hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956168 so closing down at 1732040956224 (+4 ms)Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/recovered.edits/0000000000000000003-wal.1732040956168.temp at 1732040956227 (+3 ms)3 split writer threads finished at 1732040956227Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/recovered.edits/0000000000000000003-wal.1732040956168.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732040956234 (+7 ms)Rename recovered edits hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/recovered.edits/0000000000000000003-wal.1732040956168.temp to hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/recovered.edits/0000000000000000012 at 1732040956235 (+1 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956168, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1732040956235 2024-11-19T18:29:16,237 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42267/hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956168 to hdfs://localhost:42267/hbase/oldWALs/wal.1732040956168 2024-11-19T18:29:16,238 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/recovered.edits/0000000000000000012 2024-11-19T18:29:16,533 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:29:16,536 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:29:16,549 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956536, exclude list is [], retry=0 2024-11-19T18:29:16,552 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:29:16,553 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:29:16,553 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:29:16,555 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956536 2024-11-19T18:29:16,556 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:33801:33801)] 2024-11-19T18:29:16,556 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e5c3960e9f9494c68d003384ba336f5e, NAME => 'testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:16,556 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:16,556 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,556 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,559 INFO [StoreOpener-e5c3960e9f9494c68d003384ba336f5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,561 INFO [StoreOpener-e5c3960e9f9494c68d003384ba336f5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5c3960e9f9494c68d003384ba336f5e columnFamilyName a 2024-11-19T18:29:16,561 DEBUG [StoreOpener-e5c3960e9f9494c68d003384ba336f5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:16,562 INFO [StoreOpener-e5c3960e9f9494c68d003384ba336f5e-1 {}] regionserver.HStore(327): Store=e5c3960e9f9494c68d003384ba336f5e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:16,562 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,563 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,564 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,565 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/recovered.edits/0000000000000000012 2024-11-19T18:29:16,568 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/recovered.edits/0000000000000000012 2024-11-19T18:29:16,568 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e5c3960e9f9494c68d003384ba336f5e 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-19T18:29:16,586 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/.tmp/a/0a63d6ff8730461091532a91cdcb1dcd is 79, key is testDatalossWhenInputError/a:x0/1732040956193/Put/seqid=0 2024-11-19T18:29:16,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741914_1092 (size=5808) 2024-11-19T18:29:16,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741914_1092 (size=5808) 2024-11-19T18:29:16,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741914_1092 (size=5808) 2024-11-19T18:29:16,594 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/.tmp/a/0a63d6ff8730461091532a91cdcb1dcd 2024-11-19T18:29:16,604 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/.tmp/a/0a63d6ff8730461091532a91cdcb1dcd as hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/a/0a63d6ff8730461091532a91cdcb1dcd 2024-11-19T18:29:16,613 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/a/0a63d6ff8730461091532a91cdcb1dcd, entries=10, sequenceid=12, filesize=5.7 K 2024-11-19T18:29:16,613 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for e5c3960e9f9494c68d003384ba336f5e in 45ms, sequenceid=12, compaction requested=false; wal=null 2024-11-19T18:29:16,614 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/recovered.edits/0000000000000000012 2024-11-19T18:29:16,615 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,615 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,617 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,619 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-19T18:29:16,620 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e5c3960e9f9494c68d003384ba336f5e; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60619730, jitterRate=-0.09669563174247742}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T18:29:16,621 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e5c3960e9f9494c68d003384ba336f5e: Writing region info on filesystem at 1732040956556Initializing all the Stores at 1732040956559 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040956559Obtaining lock to block concurrent updates at 1732040956568 (+9 ms)Preparing flush snapshotting stores in e5c3960e9f9494c68d003384ba336f5e at 1732040956568Finished memstore snapshotting testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732040956568Flushing stores of testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e. at 1732040956568Flushing e5c3960e9f9494c68d003384ba336f5e/a: creating writer at 1732040956568Flushing e5c3960e9f9494c68d003384ba336f5e/a: appending metadata at 1732040956586 (+18 ms)Flushing e5c3960e9f9494c68d003384ba336f5e/a: closing flushed file at 1732040956586Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4447303: reopening flushed file at 1732040956603 (+17 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for e5c3960e9f9494c68d003384ba336f5e in 45ms, sequenceid=12, compaction requested=false; wal=null at 1732040956613 (+10 ms)Cleaning up temporary data from old regions at 1732040956615 (+2 ms)Region opened successfully at 1732040956621 (+6 ms) 2024-11-19T18:29:16,624 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e5c3960e9f9494c68d003384ba336f5e, NAME => 'testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:16,624 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732040956117.e5c3960e9f9494c68d003384ba336f5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:16,624 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,624 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,625 INFO [StoreOpener-e5c3960e9f9494c68d003384ba336f5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,626 INFO [StoreOpener-e5c3960e9f9494c68d003384ba336f5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5c3960e9f9494c68d003384ba336f5e columnFamilyName a 2024-11-19T18:29:16,626 DEBUG [StoreOpener-e5c3960e9f9494c68d003384ba336f5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:16,631 DEBUG [StoreOpener-e5c3960e9f9494c68d003384ba336f5e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/a/0a63d6ff8730461091532a91cdcb1dcd 2024-11-19T18:29:16,631 INFO [StoreOpener-e5c3960e9f9494c68d003384ba336f5e-1 {}] regionserver.HStore(327): Store=e5c3960e9f9494c68d003384ba336f5e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:16,631 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,632 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,633 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,634 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,634 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,636 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e5c3960e9f9494c68d003384ba336f5e 2024-11-19T18:29:16,638 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testDatalossWhenInputError/e5c3960e9f9494c68d003384ba336f5e/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-19T18:29:16,639 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e5c3960e9f9494c68d003384ba336f5e; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59225975, jitterRate=-0.11746419966220856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T18:29:16,639 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e5c3960e9f9494c68d003384ba336f5e: Writing region info on filesystem at 1732040956624Initializing all the Stores at 1732040956625 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040956625Cleaning up temporary data from old regions at 1732040956634 (+9 ms)Region opened successfully at 1732040956639 (+5 ms) 2024-11-19T18:29:16,659 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=425 (was 415) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:59688 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1034186995-172.17.0.2-1732040924630:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:60596 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:59756 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1034186995-172.17.0.2-1732040924630:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1034186995-172.17.0.2-1732040924630:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:59656 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:59594 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1265 (was 1183) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=202 (was 202), ProcessCount=11 (was 11), AvailableMemoryMB=7048 (was 7057) 2024-11-19T18:29:16,660 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1265 is superior to 1024 2024-11-19T18:29:16,673 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=425, OpenFileDescriptor=1265, MaxFileDescriptor=1048576, SystemLoadAverage=202, ProcessCount=11, AvailableMemoryMB=7047 2024-11-19T18:29:16,673 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1265 is superior to 1024 2024-11-19T18:29:16,689 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:29:16,691 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:29:16,692 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T18:29:16,694 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-72988955, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/hregion-72988955, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:29:16,712 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-72988955/hregion-72988955.1732040956694, exclude list is [], retry=0 2024-11-19T18:29:16,717 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:29:16,719 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:29:16,721 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:29:16,740 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-72988955/hregion-72988955.1732040956694 2024-11-19T18:29:16,741 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:33801:33801)] 2024-11-19T18:29:16,741 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9c109cc76fa85548ab0ccaef0734d1c0, NAME => 'testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42267/hbase 2024-11-19T18:29:16,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741916_1094 (size=63) 2024-11-19T18:29:16,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741916_1094 (size=63) 2024-11-19T18:29:16,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741916_1094 (size=63) 2024-11-19T18:29:16,773 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:16,774 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,776 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c109cc76fa85548ab0ccaef0734d1c0 columnFamilyName a 2024-11-19T18:29:16,776 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:16,777 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(327): Store=9c109cc76fa85548ab0ccaef0734d1c0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:16,777 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,779 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c109cc76fa85548ab0ccaef0734d1c0 columnFamilyName b 2024-11-19T18:29:16,779 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:16,780 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(327): Store=9c109cc76fa85548ab0ccaef0734d1c0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:16,784 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,786 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c109cc76fa85548ab0ccaef0734d1c0 columnFamilyName c 2024-11-19T18:29:16,786 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:16,787 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(327): Store=9c109cc76fa85548ab0ccaef0734d1c0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:16,787 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,788 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,789 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,790 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,790 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,791 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:29:16,792 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,794 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T18:29:16,795 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9c109cc76fa85548ab0ccaef0734d1c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64440290, jitterRate=-0.039764851331710815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:29:16,795 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9c109cc76fa85548ab0ccaef0734d1c0: Writing region info on filesystem at 1732040956773Initializing all the Stores at 1732040956774 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040956774Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040956774Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040956774Cleaning up temporary data from old regions at 1732040956790 (+16 ms)Region opened successfully at 1732040956795 (+5 ms) 2024-11-19T18:29:16,795 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9c109cc76fa85548ab0ccaef0734d1c0, disabling compactions & flushes 2024-11-19T18:29:16,796 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0. 2024-11-19T18:29:16,796 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0. 2024-11-19T18:29:16,796 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0. after waiting 0 ms 2024-11-19T18:29:16,796 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0. 2024-11-19T18:29:16,796 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0. 2024-11-19T18:29:16,796 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9c109cc76fa85548ab0ccaef0734d1c0: Waiting for close lock at 1732040956795Disabling compacts and flushes for region at 1732040956795Disabling writes for close at 1732040956796 (+1 ms)Writing region close event to WAL at 1732040956796Closed at 1732040956796 2024-11-19T18:29:16,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741915_1093 (size=93) 2024-11-19T18:29:16,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741915_1093 (size=93) 2024-11-19T18:29:16,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741915_1093 (size=93) 2024-11-19T18:29:16,804 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T18:29:16,806 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-19T18:29:16,806 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-72988955:(num 1732040956694) 2024-11-19T18:29:16,806 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:29:16,808 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:29:16,826 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808, exclude list is [], retry=0 2024-11-19T18:29:16,829 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:29:16,829 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:29:16,829 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:29:16,831 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808 2024-11-19T18:29:16,831 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:40377:40377)] 2024-11-19T18:29:16,832 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c109cc76fa85548ab0ccaef0734d1c0, NAME => 'testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:16,832 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:16,832 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,832 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,834 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,834 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c109cc76fa85548ab0ccaef0734d1c0 columnFamilyName a 2024-11-19T18:29:16,834 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:16,835 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(327): Store=9c109cc76fa85548ab0ccaef0734d1c0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:16,835 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,836 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c109cc76fa85548ab0ccaef0734d1c0 columnFamilyName b 2024-11-19T18:29:16,836 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:16,836 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(327): Store=9c109cc76fa85548ab0ccaef0734d1c0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:16,836 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,837 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c109cc76fa85548ab0ccaef0734d1c0 columnFamilyName c 2024-11-19T18:29:16,837 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:16,838 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(327): Store=9c109cc76fa85548ab0ccaef0734d1c0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:16,838 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,839 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,840 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,841 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,841 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,841 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:29:16,843 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:16,844 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9c109cc76fa85548ab0ccaef0734d1c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65442528, jitterRate=-0.024830341339111328}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:29:16,845 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9c109cc76fa85548ab0ccaef0734d1c0: Writing region info on filesystem at 1732040956832Initializing all the Stores at 1732040956833 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040956833Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040956833Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040956833Cleaning up temporary data from old regions at 1732040956841 (+8 ms)Region opened successfully at 1732040956845 (+4 ms) 2024-11-19T18:29:16,848 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732040956848/Put/seqid=0 2024-11-19T18:29:16,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741918_1096 (size=4875) 2024-11-19T18:29:16,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741918_1096 (size=4875) 2024-11-19T18:29:16,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741918_1096 (size=4875) 2024-11-19T18:29:16,864 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732040956864/Put/seqid=0 2024-11-19T18:29:16,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741919_1097 (size=4875) 2024-11-19T18:29:16,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741919_1097 (size=4875) 2024-11-19T18:29:16,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741919_1097 (size=4875) 2024-11-19T18:29:16,882 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732040956882/Put/seqid=0 2024-11-19T18:29:16,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741920_1098 (size=4875) 2024-11-19T18:29:16,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741920_1098 (size=4875) 2024-11-19T18:29:16,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741920_1098 (size=4875) 2024-11-19T18:29:16,892 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:42267/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 9c109cc76fa85548ab0ccaef0734d1c0/a 2024-11-19T18:29:16,896 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-19T18:29:16,896 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-19T18:29:16,896 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:42267/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 9c109cc76fa85548ab0ccaef0734d1c0/a 2024-11-19T18:29:16,900 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-19T18:29:16,900 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-19T18:29:16,900 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:42267/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 9c109cc76fa85548ab0ccaef0734d1c0/a 2024-11-19T18:29:16,903 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-19T18:29:16,903 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-19T18:29:16,903 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9c109cc76fa85548ab0ccaef0734d1c0 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-19T18:29:16,922 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/.tmp/a/ee1ede88ea0643ab9faa2d39e538a4fe is 55, key is testCompactedBulkLoadedFiles/a:a/1732040956845/Put/seqid=0 2024-11-19T18:29:16,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741921_1099 (size=5107) 2024-11-19T18:29:16,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741921_1099 (size=5107) 2024-11-19T18:29:16,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741921_1099 (size=5107) 2024-11-19T18:29:16,930 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/.tmp/a/ee1ede88ea0643ab9faa2d39e538a4fe 2024-11-19T18:29:16,935 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/.tmp/a/ee1ede88ea0643ab9faa2d39e538a4fe as hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee1ede88ea0643ab9faa2d39e538a4fe 2024-11-19T18:29:16,940 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee1ede88ea0643ab9faa2d39e538a4fe, entries=1, sequenceid=4, filesize=5.0 K 2024-11-19T18:29:16,941 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 9c109cc76fa85548ab0ccaef0734d1c0 in 38ms, sequenceid=4, compaction requested=false 2024-11-19T18:29:16,941 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9c109cc76fa85548ab0ccaef0734d1c0: 2024-11-19T18:29:16,943 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/60cdb958e8e04679979e02a229767546_SeqId_4_ 2024-11-19T18:29:16,944 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee84a3fde7c04f1493ec24903b6c3c19_SeqId_4_ 2024-11-19T18:29:16,945 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/19f66a3bf8aa447bb82022eb453b0d4a_SeqId_4_ 2024-11-19T18:29:16,945 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:42267/hbase/testCompactedBulkLoadedFiles/hfile0 into 9c109cc76fa85548ab0ccaef0734d1c0/a as hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/60cdb958e8e04679979e02a229767546_SeqId_4_ - updating store file list. 2024-11-19T18:29:16,950 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 60cdb958e8e04679979e02a229767546_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-19T18:29:16,950 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/60cdb958e8e04679979e02a229767546_SeqId_4_ into 9c109cc76fa85548ab0ccaef0734d1c0/a 2024-11-19T18:29:16,950 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:42267/hbase/testCompactedBulkLoadedFiles/hfile0 into 9c109cc76fa85548ab0ccaef0734d1c0/a (new location: hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/60cdb958e8e04679979e02a229767546_SeqId_4_) 2024-11-19T18:29:16,951 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:42267/hbase/testCompactedBulkLoadedFiles/hfile1 into 9c109cc76fa85548ab0ccaef0734d1c0/a as hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee84a3fde7c04f1493ec24903b6c3c19_SeqId_4_ - updating store file list. 2024-11-19T18:29:16,955 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for ee84a3fde7c04f1493ec24903b6c3c19_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-19T18:29:16,955 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee84a3fde7c04f1493ec24903b6c3c19_SeqId_4_ into 9c109cc76fa85548ab0ccaef0734d1c0/a 2024-11-19T18:29:16,955 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:42267/hbase/testCompactedBulkLoadedFiles/hfile1 into 9c109cc76fa85548ab0ccaef0734d1c0/a (new location: hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee84a3fde7c04f1493ec24903b6c3c19_SeqId_4_) 2024-11-19T18:29:16,956 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:42267/hbase/testCompactedBulkLoadedFiles/hfile2 into 9c109cc76fa85548ab0ccaef0734d1c0/a as hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/19f66a3bf8aa447bb82022eb453b0d4a_SeqId_4_ - updating store file list. 2024-11-19T18:29:16,960 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 19f66a3bf8aa447bb82022eb453b0d4a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-19T18:29:16,960 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/19f66a3bf8aa447bb82022eb453b0d4a_SeqId_4_ into 9c109cc76fa85548ab0ccaef0734d1c0/a 2024-11-19T18:29:16,960 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:42267/hbase/testCompactedBulkLoadedFiles/hfile2 into 9c109cc76fa85548ab0ccaef0734d1c0/a (new location: hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/19f66a3bf8aa447bb82022eb453b0d4a_SeqId_4_) 2024-11-19T18:29:16,967 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T18:29:16,968 DEBUG [Time-limited test {}] regionserver.HStore(1541): 9c109cc76fa85548ab0ccaef0734d1c0/a is initiating major compaction (all files) 2024-11-19T18:29:16,968 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 9c109cc76fa85548ab0ccaef0734d1c0/a in testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0. 2024-11-19T18:29:16,968 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee1ede88ea0643ab9faa2d39e538a4fe, hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/60cdb958e8e04679979e02a229767546_SeqId_4_, hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee84a3fde7c04f1493ec24903b6c3c19_SeqId_4_, hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/19f66a3bf8aa447bb82022eb453b0d4a_SeqId_4_] into tmpdir=hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/.tmp, totalSize=19.3 K 2024-11-19T18:29:16,968 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ee1ede88ea0643ab9faa2d39e538a4fe, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732040956845 2024-11-19T18:29:16,969 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 60cdb958e8e04679979e02a229767546_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-19T18:29:16,969 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ee84a3fde7c04f1493ec24903b6c3c19_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-19T18:29:16,970 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 19f66a3bf8aa447bb82022eb453b0d4a_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-19T18:29:16,982 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/.tmp/a/2e5594d5b9ce4848a586503b080bd6ea is 55, key is testCompactedBulkLoadedFiles/a:a/1732040956845/Put/seqid=0 2024-11-19T18:29:16,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741922_1100 (size=6154) 2024-11-19T18:29:16,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741922_1100 (size=6154) 2024-11-19T18:29:16,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741922_1100 (size=6154) 2024-11-19T18:29:17,002 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/.tmp/a/2e5594d5b9ce4848a586503b080bd6ea as hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/2e5594d5b9ce4848a586503b080bd6ea 2024-11-19T18:29:17,010 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 9c109cc76fa85548ab0ccaef0734d1c0/a of 9c109cc76fa85548ab0ccaef0734d1c0 into 2e5594d5b9ce4848a586503b080bd6ea(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T18:29:17,010 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 9c109cc76fa85548ab0ccaef0734d1c0: 2024-11-19T18:29:17,010 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-19T18:29:17,011 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-19T18:29:17,052 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42267/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808, size=0 (0bytes) 2024-11-19T18:29:17,052 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:42267/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808 might be still open, length is 0 2024-11-19T18:29:17,052 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42267/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808 2024-11-19T18:29:17,052 WARN [IPC Server handler 4 on default port 42267 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-19T18:29:17,053 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42267/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808 after 1ms 2024-11-19T18:29:18,101 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-19T18:29:18,101 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-19T18:29:18,102 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-19T18:29:18,102 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-19T18:29:19,622 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:60620 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33471:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60620 dst: /127.0.0.1:33471 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33471 remote=/127.0.0.1:60620]. Total timeout mills is 60000, 57387 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:19,623 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:59808 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:41941:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59808 dst: /127.0.0.1:41941 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:19,623 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:59698 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33841:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59698 dst: /127.0.0.1:33841 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:19,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741917_1101 (size=1659) 2024-11-19T18:29:19,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741917_1101 (size=1659) 2024-11-19T18:29:21,053 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42267/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808 after 4001ms 2024-11-19T18:29:21,056 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42267/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808 took 4005ms 2024-11-19T18:29:21,058 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:42267/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808; continuing. 2024-11-19T18:29:21,058 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42267/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808 so closing down 2024-11-19T18:29:21,058 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T18:29:21,060 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732040956808.temp 2024-11-19T18:29:21,061 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/recovered.edits/0000000000000000003-wal.1732040956808.temp 2024-11-19T18:29:21,061 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T18:29:21,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741923_1102 (size=698) 2024-11-19T18:29:21,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741923_1102 (size=698) 2024-11-19T18:29:21,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741923_1102 (size=698) 2024-11-19T18:29:21,074 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/recovered.edits/0000000000000000003-wal.1732040956808.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-19T18:29:21,075 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/recovered.edits/0000000000000000003-wal.1732040956808.temp to hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/recovered.edits/0000000000000000008 2024-11-19T18:29:21,075 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 18 ms; skipped=3; WAL=hdfs://localhost:42267/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808, size=0, length=0, corrupted=false, cancelled=false 2024-11-19T18:29:21,075 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42267/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808, journal: Splitting hdfs://localhost:42267/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808, size=0 (0bytes) at 1732040957052Finishing writing output for hdfs://localhost:42267/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808 so closing down at 1732040961058 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/recovered.edits/0000000000000000003-wal.1732040956808.temp at 1732040961061 (+3 ms)3 split writer threads finished at 1732040961061Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/recovered.edits/0000000000000000003-wal.1732040956808.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732040961074 (+13 ms)Rename recovered edits hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/recovered.edits/0000000000000000003-wal.1732040956808.temp to hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/recovered.edits/0000000000000000008 at 1732040961075 (+1 ms)Processed 5 edits across 1 Regions in 18 ms; skipped=3; WAL=hdfs://localhost:42267/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808, size=0, length=0, corrupted=false, cancelled=false at 1732040961075 2024-11-19T18:29:21,077 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42267/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808 to hdfs://localhost:42267/hbase/oldWALs/wal.1732040956808 2024-11-19T18:29:21,078 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/recovered.edits/0000000000000000008 2024-11-19T18:29:21,078 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:29:21,079 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:29:21,100 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040961080, exclude list is [], retry=0 2024-11-19T18:29:21,103 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:29:21,103 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:29:21,104 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:29:21,105 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040961080 2024-11-19T18:29:21,106 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:33801:33801)] 2024-11-19T18:29:21,106 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c109cc76fa85548ab0ccaef0734d1c0, NAME => 'testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:21,106 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:21,106 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:21,106 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:21,108 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:21,108 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c109cc76fa85548ab0ccaef0734d1c0 columnFamilyName a 2024-11-19T18:29:21,109 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:21,114 DEBUG [StoreFileOpener-9c109cc76fa85548ab0ccaef0734d1c0-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 19f66a3bf8aa447bb82022eb453b0d4a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-19T18:29:21,115 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/19f66a3bf8aa447bb82022eb453b0d4a_SeqId_4_ 2024-11-19T18:29:21,118 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/2e5594d5b9ce4848a586503b080bd6ea 2024-11-19T18:29:21,122 DEBUG [StoreFileOpener-9c109cc76fa85548ab0ccaef0734d1c0-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 60cdb958e8e04679979e02a229767546_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-19T18:29:21,122 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/60cdb958e8e04679979e02a229767546_SeqId_4_ 2024-11-19T18:29:21,125 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee1ede88ea0643ab9faa2d39e538a4fe 2024-11-19T18:29:21,128 DEBUG [StoreFileOpener-9c109cc76fa85548ab0ccaef0734d1c0-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for ee84a3fde7c04f1493ec24903b6c3c19_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-19T18:29:21,128 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee84a3fde7c04f1493ec24903b6c3c19_SeqId_4_ 2024-11-19T18:29:21,128 WARN [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/19f66a3bf8aa447bb82022eb453b0d4a_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@48d5ce09 2024-11-19T18:29:21,129 WARN [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/60cdb958e8e04679979e02a229767546_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@48d5ce09 2024-11-19T18:29:21,129 WARN [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee1ede88ea0643ab9faa2d39e538a4fe from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@48d5ce09 2024-11-19T18:29:21,129 WARN [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee84a3fde7c04f1493ec24903b6c3c19_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@48d5ce09 2024-11-19T18:29:21,129 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/19f66a3bf8aa447bb82022eb453b0d4a_SeqId_4_, hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/60cdb958e8e04679979e02a229767546_SeqId_4_, hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee1ede88ea0643ab9faa2d39e538a4fe, hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee84a3fde7c04f1493ec24903b6c3c19_SeqId_4_] to archive 2024-11-19T18:29:21,130 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T18:29:21,131 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/19f66a3bf8aa447bb82022eb453b0d4a_SeqId_4_ to hdfs://localhost:42267/hbase/archive/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/19f66a3bf8aa447bb82022eb453b0d4a_SeqId_4_ 2024-11-19T18:29:21,133 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/60cdb958e8e04679979e02a229767546_SeqId_4_ to hdfs://localhost:42267/hbase/archive/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/60cdb958e8e04679979e02a229767546_SeqId_4_ 2024-11-19T18:29:21,134 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee1ede88ea0643ab9faa2d39e538a4fe to hdfs://localhost:42267/hbase/archive/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee1ede88ea0643ab9faa2d39e538a4fe 2024-11-19T18:29:21,135 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee84a3fde7c04f1493ec24903b6c3c19_SeqId_4_ to hdfs://localhost:42267/hbase/archive/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/a/ee84a3fde7c04f1493ec24903b6c3c19_SeqId_4_ 2024-11-19T18:29:21,135 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(327): Store=9c109cc76fa85548ab0ccaef0734d1c0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:21,135 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:21,136 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c109cc76fa85548ab0ccaef0734d1c0 columnFamilyName b 2024-11-19T18:29:21,136 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:21,137 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(327): Store=9c109cc76fa85548ab0ccaef0734d1c0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:21,137 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:21,137 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c109cc76fa85548ab0ccaef0734d1c0 columnFamilyName c 2024-11-19T18:29:21,137 DEBUG [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:21,138 INFO [StoreOpener-9c109cc76fa85548ab0ccaef0734d1c0-1 {}] regionserver.HStore(327): Store=9c109cc76fa85548ab0ccaef0734d1c0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:21,138 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:21,139 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:21,140 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:21,141 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/recovered.edits/0000000000000000008 2024-11-19T18:29:21,145 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 9c109cc76fa85548ab0ccaef0734d1c0 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "9c109cc76fa85548ab0ccaef0734d1c0" family_name: "a" compaction_input: "ee1ede88ea0643ab9faa2d39e538a4fe" compaction_input: "60cdb958e8e04679979e02a229767546_SeqId_4_" compaction_input: "ee84a3fde7c04f1493ec24903b6c3c19_SeqId_4_" compaction_input: "19f66a3bf8aa447bb82022eb453b0d4a_SeqId_4_" compaction_output: "2e5594d5b9ce4848a586503b080bd6ea" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-19T18:29:21,145 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-19T18:29:21,145 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/recovered.edits/0000000000000000008 2024-11-19T18:29:21,146 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/recovered.edits/0000000000000000008 2024-11-19T18:29:21,146 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:21,147 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:21,147 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:29:21,149 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9c109cc76fa85548ab0ccaef0734d1c0 2024-11-19T18:29:21,151 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testCompactedBulkLoadedFiles/9c109cc76fa85548ab0ccaef0734d1c0/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-19T18:29:21,152 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9c109cc76fa85548ab0ccaef0734d1c0; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66016280, jitterRate=-0.016280770301818848}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:29:21,152 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9c109cc76fa85548ab0ccaef0734d1c0: Writing region info on filesystem at 1732040961106Initializing all the Stores at 1732040961107 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040961107Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040961108 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040961108Cleaning up temporary data from old regions at 1732040961147 (+39 ms)Region opened successfully at 1732040961152 (+5 ms) 2024-11-19T18:29:21,154 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9c109cc76fa85548ab0ccaef0734d1c0, disabling compactions & flushes 2024-11-19T18:29:21,154 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0. 2024-11-19T18:29:21,154 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0. 2024-11-19T18:29:21,154 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0. after waiting 0 ms 2024-11-19T18:29:21,154 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0. 2024-11-19T18:29:21,155 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732040956690.9c109cc76fa85548ab0ccaef0734d1c0. 2024-11-19T18:29:21,155 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9c109cc76fa85548ab0ccaef0734d1c0: Waiting for close lock at 1732040961154Disabling compacts and flushes for region at 1732040961154Disabling writes for close at 1732040961154Writing region close event to WAL at 1732040961155 (+1 ms)Closed at 1732040961155 2024-11-19T18:29:21,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741924_1103 (size=93) 2024-11-19T18:29:21,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741924_1103 (size=93) 2024-11-19T18:29:21,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741924_1103 (size=93) 2024-11-19T18:29:21,160 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-19T18:29:21,160 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732040961080) 2024-11-19T18:29:21,176 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=438 (was 425) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-760636024_22 at /127.0.0.1:53890 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-760636024_22 at /127.0.0.1:55592 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:42267 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:42267 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-760636024_22 at /127.0.0.1:39322 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1265) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=274 (was 202) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6988 (was 7047) 2024-11-19T18:29:21,176 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-19T18:29:21,189 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=438, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=274, ProcessCount=11, AvailableMemoryMB=6986 2024-11-19T18:29:21,189 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-19T18:29:21,204 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:29:21,206 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T18:29:21,206 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T18:29:21,209 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-91823742, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/hregion-91823742, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:29:21,222 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-91823742/hregion-91823742.1732040961209, exclude list is [], retry=0 2024-11-19T18:29:21,225 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:29:21,226 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:29:21,226 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:29:21,228 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-91823742/hregion-91823742.1732040961209 2024-11-19T18:29:21,228 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:37769:37769)] 2024-11-19T18:29:21,229 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b81043a5af427966ec37810f781fd56b, NAME => 'testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42267/hbase 2024-11-19T18:29:21,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741926_1105 (size=67) 2024-11-19T18:29:21,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741926_1105 (size=67) 2024-11-19T18:29:21,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741926_1105 (size=67) 2024-11-19T18:29:21,237 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:21,238 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,240 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b81043a5af427966ec37810f781fd56b columnFamilyName a 2024-11-19T18:29:21,240 DEBUG [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:21,240 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(327): Store=b81043a5af427966ec37810f781fd56b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:21,241 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,242 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b81043a5af427966ec37810f781fd56b columnFamilyName b 2024-11-19T18:29:21,242 DEBUG [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:21,242 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(327): Store=b81043a5af427966ec37810f781fd56b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:21,243 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,244 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b81043a5af427966ec37810f781fd56b columnFamilyName c 2024-11-19T18:29:21,244 DEBUG [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:21,244 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(327): Store=b81043a5af427966ec37810f781fd56b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:21,244 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,245 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,245 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,246 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,246 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,247 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:29:21,248 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,250 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T18:29:21,251 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b81043a5af427966ec37810f781fd56b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72606568, jitterRate=0.08192217350006104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:29:21,251 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b81043a5af427966ec37810f781fd56b: Writing region info on filesystem at 1732040961237Initializing all the Stores at 1732040961238 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040961238Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040961238Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040961238Cleaning up temporary data from old regions at 1732040961246 (+8 ms)Region opened successfully at 1732040961251 (+5 ms) 2024-11-19T18:29:21,251 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b81043a5af427966ec37810f781fd56b, disabling compactions & flushes 2024-11-19T18:29:21,251 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. 2024-11-19T18:29:21,251 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. 2024-11-19T18:29:21,251 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. after waiting 0 ms 2024-11-19T18:29:21,251 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. 2024-11-19T18:29:21,252 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. 2024-11-19T18:29:21,252 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b81043a5af427966ec37810f781fd56b: Waiting for close lock at 1732040961251Disabling compacts and flushes for region at 1732040961251Disabling writes for close at 1732040961251Writing region close event to WAL at 1732040961252 (+1 ms)Closed at 1732040961252 2024-11-19T18:29:21,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741925_1104 (size=93) 2024-11-19T18:29:21,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741925_1104 (size=93) 2024-11-19T18:29:21,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741925_1104 (size=93) 2024-11-19T18:29:21,256 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-19T18:29:21,256 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-91823742:(num 1732040961209) 2024-11-19T18:29:21,257 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:29:21,258 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:29:21,279 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961259, exclude list is [], retry=0 2024-11-19T18:29:21,282 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:29:21,282 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:29:21,283 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:29:21,284 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961259 2024-11-19T18:29:21,285 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:40377:40377)] 2024-11-19T18:29:21,285 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b81043a5af427966ec37810f781fd56b, NAME => 'testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:21,285 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:21,285 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,285 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,286 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,287 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b81043a5af427966ec37810f781fd56b columnFamilyName a 2024-11-19T18:29:21,287 DEBUG [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:21,288 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(327): Store=b81043a5af427966ec37810f781fd56b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:21,288 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,289 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b81043a5af427966ec37810f781fd56b columnFamilyName b 2024-11-19T18:29:21,289 DEBUG [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:21,289 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(327): Store=b81043a5af427966ec37810f781fd56b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:21,290 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,290 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b81043a5af427966ec37810f781fd56b columnFamilyName c 2024-11-19T18:29:21,290 DEBUG [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:21,291 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(327): Store=b81043a5af427966ec37810f781fd56b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:21,291 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,292 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,293 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,294 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,294 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,294 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:29:21,296 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,297 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b81043a5af427966ec37810f781fd56b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58727321, jitterRate=-0.12489472329616547}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:29:21,297 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b81043a5af427966ec37810f781fd56b: Writing region info on filesystem at 1732040961285Initializing all the Stores at 1732040961286 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040961286Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040961286Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040961286Cleaning up temporary data from old regions at 1732040961294 (+8 ms)Region opened successfully at 1732040961297 (+3 ms) 2024-11-19T18:29:21,304 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b81043a5af427966ec37810f781fd56b 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-19T18:29:21,321 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/a/dba691c4a13f4f36bf91cf87108cf785 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732040961297/Put/seqid=0 2024-11-19T18:29:21,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741928_1107 (size=5958) 2024-11-19T18:29:21,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741928_1107 (size=5958) 2024-11-19T18:29:21,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741928_1107 (size=5958) 2024-11-19T18:29:21,328 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/a/dba691c4a13f4f36bf91cf87108cf785 2024-11-19T18:29:21,334 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/a/dba691c4a13f4f36bf91cf87108cf785 as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/a/dba691c4a13f4f36bf91cf87108cf785 2024-11-19T18:29:21,338 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/a/dba691c4a13f4f36bf91cf87108cf785, entries=10, sequenceid=13, filesize=5.8 K 2024-11-19T18:29:21,340 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for b81043a5af427966ec37810f781fd56b in 35ms, sequenceid=13, compaction requested=false 2024-11-19T18:29:21,340 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b81043a5af427966ec37810f781fd56b: 2024-11-19T18:29:21,355 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b81043a5af427966ec37810f781fd56b, disabling compactions & flushes 2024-11-19T18:29:21,356 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. 2024-11-19T18:29:21,356 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. 2024-11-19T18:29:21,356 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. after waiting 0 ms 2024-11-19T18:29:21,356 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. 2024-11-19T18:29:21,356 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. 2024-11-19T18:29:21,356 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. 2024-11-19T18:29:21,356 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b81043a5af427966ec37810f781fd56b: Waiting for close lock at 1732040961355Disabling compacts and flushes for region at 1732040961355Disabling writes for close at 1732040961356 (+1 ms)Writing region close event to WAL at 1732040961356Closed at 1732040961356 2024-11-19T18:29:21,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741927_1106 (size=5869) 2024-11-19T18:29:21,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741927_1106 (size=5869) 2024-11-19T18:29:21,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741927_1106 (size=5869) 2024-11-19T18:29:21,375 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961259, size=5.7 K (5869bytes) 2024-11-19T18:29:21,375 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961259 2024-11-19T18:29:21,375 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961259 after 0ms 2024-11-19T18:29:21,377 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961259 took 2ms 2024-11-19T18:29:21,379 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961259 so closing down 2024-11-19T18:29:21,379 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T18:29:21,380 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732040961259.temp 2024-11-19T18:29:21,381 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000003-wal.1732040961259.temp 2024-11-19T18:29:21,381 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T18:29:21,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741929_1108 (size=5223) 2024-11-19T18:29:21,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741929_1108 (size=5223) 2024-11-19T18:29:21,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741929_1108 (size=5223) 2024-11-19T18:29:21,390 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000003-wal.1732040961259.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-19T18:29:21,391 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000003-wal.1732040961259.temp to hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000035 2024-11-19T18:29:21,392 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961259, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-11-19T18:29:21,392 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961259, journal: Splitting hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961259, size=5.7 K (5869bytes) at 1732040961375Finishing writing output for hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961259 so closing down at 1732040961379 (+4 ms)Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000003-wal.1732040961259.temp at 1732040961381 (+2 ms)3 split writer threads finished at 1732040961381Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000003-wal.1732040961259.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732040961390 (+9 ms)Rename recovered edits hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000003-wal.1732040961259.temp to hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000035 at 1732040961391 (+1 ms)Processed 32 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961259, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1732040961392 (+1 ms) 2024-11-19T18:29:21,393 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961259 to hdfs://localhost:42267/hbase/oldWALs/wal.1732040961259 2024-11-19T18:29:21,394 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000035 2024-11-19T18:29:21,394 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:29:21,395 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:29:21,402 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T18:29:21,409 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396, exclude list is [], retry=0 2024-11-19T18:29:21,412 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:29:21,412 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:29:21,412 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:29:21,414 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396 2024-11-19T18:29:21,415 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:40377:40377),(127.0.0.1/127.0.0.1:33801:33801)] 2024-11-19T18:29:21,415 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b81043a5af427966ec37810f781fd56b, NAME => 'testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b.', STARTKEY => '', ENDKEY => ''} 2024-11-19T18:29:21,415 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:21,415 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,415 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,419 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,420 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b81043a5af427966ec37810f781fd56b columnFamilyName a 2024-11-19T18:29:21,420 DEBUG [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:21,427 DEBUG [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/a/dba691c4a13f4f36bf91cf87108cf785 2024-11-19T18:29:21,428 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(327): Store=b81043a5af427966ec37810f781fd56b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:21,428 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,429 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b81043a5af427966ec37810f781fd56b columnFamilyName b 2024-11-19T18:29:21,429 DEBUG [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:21,429 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(327): Store=b81043a5af427966ec37810f781fd56b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:21,429 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,430 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b81043a5af427966ec37810f781fd56b columnFamilyName c 2024-11-19T18:29:21,430 DEBUG [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:21,430 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(327): Store=b81043a5af427966ec37810f781fd56b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:21,430 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,431 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,432 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,433 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000035 2024-11-19T18:29:21,436 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000035 2024-11-19T18:29:21,436 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b81043a5af427966ec37810f781fd56b 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-19T18:29:21,452 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/b/df74596bc0174becb7e0bfded39284b6 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732040961340/Put/seqid=0 2024-11-19T18:29:21,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741931_1110 (size=5958) 2024-11-19T18:29:21,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741931_1110 (size=5958) 2024-11-19T18:29:21,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741931_1110 (size=5958) 2024-11-19T18:29:21,461 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/b/df74596bc0174becb7e0bfded39284b6 2024-11-19T18:29:21,483 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/c/e0787b534c6b4278b376d1c43aeee3e9 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732040961346/Put/seqid=0 2024-11-19T18:29:21,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741932_1111 (size=5958) 2024-11-19T18:29:21,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741932_1111 (size=5958) 2024-11-19T18:29:21,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741932_1111 (size=5958) 2024-11-19T18:29:21,490 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/c/e0787b534c6b4278b376d1c43aeee3e9 2024-11-19T18:29:21,496 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/b/df74596bc0174becb7e0bfded39284b6 as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/b/df74596bc0174becb7e0bfded39284b6 2024-11-19T18:29:21,502 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/b/df74596bc0174becb7e0bfded39284b6, entries=10, sequenceid=35, filesize=5.8 K 2024-11-19T18:29:21,503 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/c/e0787b534c6b4278b376d1c43aeee3e9 as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/c/e0787b534c6b4278b376d1c43aeee3e9 2024-11-19T18:29:21,507 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/c/e0787b534c6b4278b376d1c43aeee3e9, entries=10, sequenceid=35, filesize=5.8 K 2024-11-19T18:29:21,507 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for b81043a5af427966ec37810f781fd56b in 71ms, sequenceid=35, compaction requested=false; wal=null 2024-11-19T18:29:21,508 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000035 2024-11-19T18:29:21,509 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,509 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,510 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:29:21,511 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:21,513 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-19T18:29:21,514 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b81043a5af427966ec37810f781fd56b; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63366687, jitterRate=-0.05576278269290924}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:29:21,514 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b81043a5af427966ec37810f781fd56b: Writing region info on filesystem at 1732040961415Initializing all the Stores at 1732040961416 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040961416Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040961418 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040961418Obtaining lock to block concurrent updates at 1732040961436 (+18 ms)Preparing flush snapshotting stores in b81043a5af427966ec37810f781fd56b at 1732040961436Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732040961436Flushing stores of testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. at 1732040961436Flushing b81043a5af427966ec37810f781fd56b/b: creating writer at 1732040961437 (+1 ms)Flushing b81043a5af427966ec37810f781fd56b/b: appending metadata at 1732040961451 (+14 ms)Flushing b81043a5af427966ec37810f781fd56b/b: closing flushed file at 1732040961451Flushing b81043a5af427966ec37810f781fd56b/c: creating writer at 1732040961467 (+16 ms)Flushing b81043a5af427966ec37810f781fd56b/c: appending metadata at 1732040961482 (+15 ms)Flushing b81043a5af427966ec37810f781fd56b/c: closing flushed file at 1732040961482Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7682b7f6: reopening flushed file at 1732040961496 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12ec3ec1: reopening flushed file at 1732040961502 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for b81043a5af427966ec37810f781fd56b in 71ms, sequenceid=35, compaction requested=false; wal=null at 1732040961507 (+5 ms)Cleaning up temporary data from old regions at 1732040961509 (+2 ms)Region opened successfully at 1732040961514 (+5 ms) 2024-11-19T18:29:21,583 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396, size=0 (0bytes) 2024-11-19T18:29:21,583 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396 might be still open, length is 0 2024-11-19T18:29:21,583 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396 2024-11-19T18:29:21,584 WARN [IPC Server handler 1 on default port 42267 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-19T18:29:21,584 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396 after 1ms 2024-11-19T18:29:22,621 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:55638 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33841:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55638 dst: /127.0.0.1:33841 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33841 remote=/127.0.0.1:55638]. Total timeout mills is 60000, 58921 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:22,621 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:53972 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:41941:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53972 dst: /127.0.0.1:41941 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:22,622 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1454744767_22 at /127.0.0.1:39362 [Receiving block BP-1034186995-172.17.0.2-1732040924630:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33471:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39362 dst: /127.0.0.1:33471 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:22,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741930_1112 (size=5215) 2024-11-19T18:29:22,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741930_1112 (size=5215) 2024-11-19T18:29:22,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741930_1112 (size=5215) 2024-11-19T18:29:25,585 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396 after 4002ms 2024-11-19T18:29:25,588 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396 took 4005ms 2024-11-19T18:29:25,590 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396; continuing. 2024-11-19T18:29:25,590 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396 so closing down 2024-11-19T18:29:25,590 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T18:29:25,591 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732040961396.temp 2024-11-19T18:29:25,593 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000037-wal.1732040961396.temp 2024-11-19T18:29:25,593 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T18:29:25,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741933_1113 (size=5223) 2024-11-19T18:29:25,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741933_1113 (size=5223) 2024-11-19T18:29:25,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741933_1113 (size=5223) 2024-11-19T18:29:25,599 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000037-wal.1732040961396.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-19T18:29:25,601 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000037-wal.1732040961396.temp to hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000066 2024-11-19T18:29:25,601 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396, size=0, length=0, corrupted=false, cancelled=false 2024-11-19T18:29:25,601 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396, journal: Splitting hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396, size=0 (0bytes) at 1732040961583Finishing writing output for hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396 so closing down at 1732040965590 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000037-wal.1732040961396.temp at 1732040965593 (+3 ms)3 split writer threads finished at 1732040965593Closed recovered edits writer path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000037-wal.1732040961396.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732040965599 (+6 ms)Rename recovered edits hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000037-wal.1732040961396.temp to hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000066 at 1732040965601 (+2 ms)Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396, size=0, length=0, corrupted=false, cancelled=false at 1732040965601 2024-11-19T18:29:25,603 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396 to hdfs://localhost:42267/hbase/oldWALs/wal.1732040961396 2024-11-19T18:29:25,604 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000066 2024-11-19T18:29:25,604 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-19T18:29:25,606 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42267/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204, archiveDir=hdfs://localhost:42267/hbase/oldWALs, maxLogs=32 2024-11-19T18:29:25,619 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040965606, exclude list is [], retry=0 2024-11-19T18:29:25,622 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33471,DS-1031e5de-a507-4f01-9ea9-9f59931468fa,DISK] 2024-11-19T18:29:25,622 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33841,DS-e43a1924-a450-49c6-ae8c-1ac234131af5,DISK] 2024-11-19T18:29:25,623 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41941,DS-3a36223b-fdd9-48e6-8d2b-3ac33c5de1dc,DISK] 2024-11-19T18:29:25,624 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040965606 2024-11-19T18:29:25,627 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33801:33801),(127.0.0.1/127.0.0.1:37769:37769),(127.0.0.1/127.0.0.1:40377:40377)] 2024-11-19T18:29:25,628 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T18:29:25,629 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:25,630 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b81043a5af427966ec37810f781fd56b columnFamilyName a 2024-11-19T18:29:25,630 DEBUG [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:25,637 DEBUG [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/a/dba691c4a13f4f36bf91cf87108cf785 2024-11-19T18:29:25,637 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(327): Store=b81043a5af427966ec37810f781fd56b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:25,637 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:25,638 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b81043a5af427966ec37810f781fd56b columnFamilyName b 2024-11-19T18:29:25,638 DEBUG [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:25,643 DEBUG [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/b/df74596bc0174becb7e0bfded39284b6 2024-11-19T18:29:25,643 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(327): Store=b81043a5af427966ec37810f781fd56b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:25,643 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:25,644 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b81043a5af427966ec37810f781fd56b columnFamilyName c 2024-11-19T18:29:25,644 DEBUG [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T18:29:25,649 DEBUG [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/c/e0787b534c6b4278b376d1c43aeee3e9 2024-11-19T18:29:25,649 INFO [StoreOpener-b81043a5af427966ec37810f781fd56b-1 {}] regionserver.HStore(327): Store=b81043a5af427966ec37810f781fd56b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T18:29:25,649 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:25,650 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:25,651 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:25,652 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000066 2024-11-19T18:29:25,659 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000066 2024-11-19T18:29:25,659 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b81043a5af427966ec37810f781fd56b 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-19T18:29:25,675 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/a/2b1cd6f7ef8e44b88bf7ea4260c4ce6f is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732040961521/Put/seqid=0 2024-11-19T18:29:25,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741935_1115 (size=5958) 2024-11-19T18:29:25,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741935_1115 (size=5958) 2024-11-19T18:29:25,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741935_1115 (size=5958) 2024-11-19T18:29:25,682 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/a/2b1cd6f7ef8e44b88bf7ea4260c4ce6f 2024-11-19T18:29:25,701 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/b/98dd7a85dde2403588ccba0c177a4ee8 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732040961528/Put/seqid=0 2024-11-19T18:29:25,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741936_1116 (size=5958) 2024-11-19T18:29:25,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741936_1116 (size=5958) 2024-11-19T18:29:25,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741936_1116 (size=5958) 2024-11-19T18:29:25,708 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/b/98dd7a85dde2403588ccba0c177a4ee8 2024-11-19T18:29:25,736 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/c/8a5455d17db34284bad47373ebe5b190 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732040961535/Put/seqid=0 2024-11-19T18:29:25,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741937_1117 (size=5958) 2024-11-19T18:29:25,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741937_1117 (size=5958) 2024-11-19T18:29:25,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741937_1117 (size=5958) 2024-11-19T18:29:25,743 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/c/8a5455d17db34284bad47373ebe5b190 2024-11-19T18:29:25,749 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/a/2b1cd6f7ef8e44b88bf7ea4260c4ce6f as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/a/2b1cd6f7ef8e44b88bf7ea4260c4ce6f 2024-11-19T18:29:25,754 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/a/2b1cd6f7ef8e44b88bf7ea4260c4ce6f, entries=10, sequenceid=66, filesize=5.8 K 2024-11-19T18:29:25,754 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/b/98dd7a85dde2403588ccba0c177a4ee8 as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/b/98dd7a85dde2403588ccba0c177a4ee8 2024-11-19T18:29:25,759 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/b/98dd7a85dde2403588ccba0c177a4ee8, entries=10, sequenceid=66, filesize=5.8 K 2024-11-19T18:29:25,760 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/.tmp/c/8a5455d17db34284bad47373ebe5b190 as hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/c/8a5455d17db34284bad47373ebe5b190 2024-11-19T18:29:25,765 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/c/8a5455d17db34284bad47373ebe5b190, entries=10, sequenceid=66, filesize=5.8 K 2024-11-19T18:29:25,766 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for b81043a5af427966ec37810f781fd56b in 107ms, sequenceid=66, compaction requested=false; wal=null 2024-11-19T18:29:25,766 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/0000000000000000066 2024-11-19T18:29:25,768 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:25,768 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:25,768 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T18:29:25,770 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b81043a5af427966ec37810f781fd56b 2024-11-19T18:29:25,772 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/hbase/data/default/testReplayEditsWrittenViaHRegion/b81043a5af427966ec37810f781fd56b/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-19T18:29:25,773 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b81043a5af427966ec37810f781fd56b; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75222889, jitterRate=0.12090839445590973}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T18:29:25,773 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b81043a5af427966ec37810f781fd56b: Writing region info on filesystem at 1732040965628Initializing all the Stores at 1732040965629 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040965629Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040965629Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732040965629Obtaining lock to block concurrent updates at 1732040965659 (+30 ms)Preparing flush snapshotting stores in b81043a5af427966ec37810f781fd56b at 1732040965659Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732040965659Flushing stores of testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. at 1732040965659Flushing b81043a5af427966ec37810f781fd56b/a: creating writer at 1732040965660 (+1 ms)Flushing b81043a5af427966ec37810f781fd56b/a: appending metadata at 1732040965674 (+14 ms)Flushing b81043a5af427966ec37810f781fd56b/a: closing flushed file at 1732040965674Flushing b81043a5af427966ec37810f781fd56b/b: creating writer at 1732040965687 (+13 ms)Flushing b81043a5af427966ec37810f781fd56b/b: appending metadata at 1732040965701 (+14 ms)Flushing b81043a5af427966ec37810f781fd56b/b: closing flushed file at 1732040965701Flushing b81043a5af427966ec37810f781fd56b/c: creating writer at 1732040965713 (+12 ms)Flushing b81043a5af427966ec37810f781fd56b/c: appending metadata at 1732040965735 (+22 ms)Flushing b81043a5af427966ec37810f781fd56b/c: closing flushed file at 1732040965735Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3789885e: reopening flushed file at 1732040965748 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2453116e: reopening flushed file at 1732040965754 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38a1b2e5: reopening flushed file at 1732040965760 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for b81043a5af427966ec37810f781fd56b in 107ms, sequenceid=66, compaction requested=false; wal=null at 1732040965766 (+6 ms)Cleaning up temporary data from old regions at 1732040965768 (+2 ms)Region opened successfully at 1732040965773 (+5 ms) 2024-11-19T18:29:25,788 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b81043a5af427966ec37810f781fd56b, disabling compactions & flushes 2024-11-19T18:29:25,788 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. 2024-11-19T18:29:25,788 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. 2024-11-19T18:29:25,788 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. after waiting 0 ms 2024-11-19T18:29:25,788 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. 2024-11-19T18:29:25,789 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732040961205.b81043a5af427966ec37810f781fd56b. 2024-11-19T18:29:25,790 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b81043a5af427966ec37810f781fd56b: Waiting for close lock at 1732040965788Disabling compacts and flushes for region at 1732040965788Disabling writes for close at 1732040965788Writing region close event to WAL at 1732040965789 (+1 ms)Closed at 1732040965789 2024-11-19T18:29:25,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741934_1114 (size=93) 2024-11-19T18:29:25,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741934_1114 (size=93) 2024-11-19T18:29:25,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741934_1114 (size=93) 2024-11-19T18:29:25,794 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-19T18:29:25,794 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732040965606) 2024-11-19T18:29:25,811 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=445 (was 438) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1402094145_22 at /127.0.0.1:39394 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:42267 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:42267 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1402094145_22 at /127.0.0.1:55708 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1402094145_22 at /127.0.0.1:54002 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 274), ProcessCount=11 (was 11), AvailableMemoryMB=6959 (was 6986) 2024-11-19T18:29:25,811 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-19T18:29:25,811 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T18:29:25,811 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T18:29:25,812 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T18:29:25,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T18:29:25,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T18:29:25,812 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T18:29:25,813 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T18:29:25,813 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1017778761, stopped=false 2024-11-19T18:29:25,813 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=30db5f576be8,42887,1732040927686 2024-11-19T18:29:25,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T18:29:25,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T18:29:25,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:29:25,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T18:29:25,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:29:25,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:29:25,815 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T18:29:25,816 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T18:29:25,816 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T18:29:25,816 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T18:29:25,816 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T18:29:25,816 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T18:29:25,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T18:29:25,817 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '30db5f576be8,38055,1732040928383' ***** 2024-11-19T18:29:25,817 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T18:29:25,817 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '30db5f576be8,40263,1732040928486' ***** 2024-11-19T18:29:25,817 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T18:29:25,817 INFO [RS:0;30db5f576be8:38055 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T18:29:25,817 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T18:29:25,817 INFO [RS:0;30db5f576be8:38055 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T18:29:25,817 INFO [RS:0;30db5f576be8:38055 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T18:29:25,817 INFO [RS:1;30db5f576be8:40263 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T18:29:25,817 INFO [RS:0;30db5f576be8:38055 {}] regionserver.HRegionServer(959): stopping server 30db5f576be8,38055,1732040928383 2024-11-19T18:29:25,817 INFO [RS:0;30db5f576be8:38055 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T18:29:25,818 INFO [RS:1;30db5f576be8:40263 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T18:29:25,818 INFO [RS:0;30db5f576be8:38055 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;30db5f576be8:38055. 2024-11-19T18:29:25,818 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T18:29:25,818 INFO [RS:1;30db5f576be8:40263 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T18:29:25,818 DEBUG [RS:0;30db5f576be8:38055 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T18:29:25,818 INFO [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(3091): Received CLOSE for 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:25,818 DEBUG [RS:0;30db5f576be8:38055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T18:29:25,818 INFO [RS:0;30db5f576be8:38055 {}] regionserver.HRegionServer(976): stopping server 30db5f576be8,38055,1732040928383; all regions closed. 2024-11-19T18:29:25,818 INFO [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(959): stopping server 30db5f576be8,40263,1732040928486 2024-11-19T18:29:25,818 INFO [RS:1;30db5f576be8:40263 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T18:29:25,818 INFO [RS:1;30db5f576be8:40263 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;30db5f576be8:40263. 2024-11-19T18:29:25,818 DEBUG [RS:1;30db5f576be8:40263 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T18:29:25,818 DEBUG [RS:1;30db5f576be8:40263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T18:29:25,818 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 74fc47bc7ee81b8f5f6b4b68f2317277, disabling compactions & flushes 2024-11-19T18:29:25,818 INFO [RS:1;30db5f576be8:40263 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T18:29:25,818 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:25,818 INFO [RS:1;30db5f576be8:40263 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T18:29:25,818 INFO [RS:1;30db5f576be8:40263 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T18:29:25,818 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:25,818 INFO [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T18:29:25,818 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. after waiting 0 ms 2024-11-19T18:29:25,818 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:25,819 INFO [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-19T18:29:25,819 DEBUG [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 74fc47bc7ee81b8f5f6b4b68f2317277=testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277.} 2024-11-19T18:29:25,819 DEBUG [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 74fc47bc7ee81b8f5f6b4b68f2317277 2024-11-19T18:29:25,819 DEBUG [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T18:29:25,819 INFO [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T18:29:25,819 DEBUG [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T18:29:25,819 DEBUG [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T18:29:25,819 DEBUG [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T18:29:25,819 INFO [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-19T18:29:25,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741833_1009 (size=2598) 2024-11-19T18:29:25,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741833_1009 (size=2598) 2024-11-19T18:29:25,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741833_1009 (size=2598) 2024-11-19T18:29:25,822 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/WALs/30db5f576be8,38055,1732040928383/30db5f576be8%2C38055%2C1732040928383.1732040929838 not finished, retry = 0 2024-11-19T18:29:25,824 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/default/testReplayEditsAfterRegionMovedWithMultiCF/74fc47bc7ee81b8f5f6b4b68f2317277/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-19T18:29:25,825 INFO [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:25,825 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 74fc47bc7ee81b8f5f6b4b68f2317277: Waiting for close lock at 1732040965818Running coprocessor pre-close hooks at 1732040965818Disabling compacts and flushes for region at 1732040965818Disabling writes for close at 1732040965818Writing region close event to WAL at 1732040965820 (+2 ms)Running coprocessor post-close hooks at 1732040965825 (+5 ms)Closed at 1732040965825 2024-11-19T18:29:25,825 DEBUG [RS_CLOSE_REGION-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277. 2024-11-19T18:29:25,842 DEBUG [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740/.tmp/info/13e4a7e0e31f4e6bbe8c333c8ea84873 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732040942659.74fc47bc7ee81b8f5f6b4b68f2317277./info:regioninfo/1732040945740/Put/seqid=0 2024-11-19T18:29:25,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741938_1118 (size=8243) 2024-11-19T18:29:25,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741938_1118 (size=8243) 2024-11-19T18:29:25,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741938_1118 (size=8243) 2024-11-19T18:29:25,849 INFO [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740/.tmp/info/13e4a7e0e31f4e6bbe8c333c8ea84873 2024-11-19T18:29:25,870 DEBUG [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740/.tmp/ns/1933840654cc4a5eae348e37a3e07c70 is 43, key is default/ns:d/1732040930663/Put/seqid=0 2024-11-19T18:29:25,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741939_1119 (size=5153) 2024-11-19T18:29:25,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741939_1119 (size=5153) 2024-11-19T18:29:25,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741939_1119 (size=5153) 2024-11-19T18:29:25,877 INFO [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740/.tmp/ns/1933840654cc4a5eae348e37a3e07c70 2024-11-19T18:29:25,897 DEBUG [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740/.tmp/table/f536d38783044397bc16b32594f418fc is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732040943090/Put/seqid=0 2024-11-19T18:29:25,901 INFO [regionserver/30db5f576be8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T18:29:25,901 INFO [regionserver/30db5f576be8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T18:29:25,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741940_1120 (size=5431) 2024-11-19T18:29:25,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741940_1120 (size=5431) 2024-11-19T18:29:25,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741940_1120 (size=5431) 2024-11-19T18:29:25,904 INFO [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740/.tmp/table/f536d38783044397bc16b32594f418fc 2024-11-19T18:29:25,910 DEBUG [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740/.tmp/info/13e4a7e0e31f4e6bbe8c333c8ea84873 as hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740/info/13e4a7e0e31f4e6bbe8c333c8ea84873 2024-11-19T18:29:25,915 INFO [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740/info/13e4a7e0e31f4e6bbe8c333c8ea84873, entries=18, sequenceid=21, filesize=8.0 K 2024-11-19T18:29:25,916 DEBUG [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740/.tmp/ns/1933840654cc4a5eae348e37a3e07c70 as hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740/ns/1933840654cc4a5eae348e37a3e07c70 2024-11-19T18:29:25,920 INFO [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740/ns/1933840654cc4a5eae348e37a3e07c70, entries=2, sequenceid=21, filesize=5.0 K 2024-11-19T18:29:25,921 DEBUG [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740/.tmp/table/f536d38783044397bc16b32594f418fc as hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740/table/f536d38783044397bc16b32594f418fc 2024-11-19T18:29:25,924 DEBUG [RS:0;30db5f576be8:38055 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/oldWALs 2024-11-19T18:29:25,924 INFO [RS:0;30db5f576be8:38055 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 30db5f576be8%2C38055%2C1732040928383:(num 1732040929838) 2024-11-19T18:29:25,924 DEBUG [RS:0;30db5f576be8:38055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T18:29:25,924 INFO [RS:0;30db5f576be8:38055 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T18:29:25,924 INFO [RS:0;30db5f576be8:38055 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T18:29:25,924 INFO [RS:0;30db5f576be8:38055 {}] hbase.ChoreService(370): Chore service for: regionserver/30db5f576be8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-19T18:29:25,924 INFO [RS:0;30db5f576be8:38055 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T18:29:25,924 INFO [RS:0;30db5f576be8:38055 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T18:29:25,924 INFO [regionserver/30db5f576be8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T18:29:25,924 INFO [RS:0;30db5f576be8:38055 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T18:29:25,925 INFO [RS:0;30db5f576be8:38055 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T18:29:25,925 INFO [RS:0;30db5f576be8:38055 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38055 2024-11-19T18:29:25,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/30db5f576be8,38055,1732040928383 2024-11-19T18:29:25,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T18:29:25,927 INFO [RS:0;30db5f576be8:38055 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T18:29:25,927 INFO [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740/table/f536d38783044397bc16b32594f418fc, entries=2, sequenceid=21, filesize=5.3 K 2024-11-19T18:29:25,928 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [30db5f576be8,38055,1732040928383] 2024-11-19T18:29:25,928 INFO [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 109ms, sequenceid=21, compaction requested=false 2024-11-19T18:29:25,932 DEBUG [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-19T18:29:25,933 DEBUG [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T18:29:25,933 INFO [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T18:29:25,933 DEBUG [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732040965819Running coprocessor pre-close hooks at 1732040965819Disabling compacts and flushes for region at 1732040965819Disabling writes for close at 1732040965819Obtaining lock to block concurrent updates at 1732040965819Preparing flush snapshotting stores in 1588230740 at 1732040965819Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1732040965820 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732040965821 (+1 ms)Flushing 1588230740/info: creating writer at 1732040965822 (+1 ms)Flushing 1588230740/info: appending metadata at 1732040965841 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732040965842 (+1 ms)Flushing 1588230740/ns: creating writer at 1732040965855 (+13 ms)Flushing 1588230740/ns: appending metadata at 1732040965870 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732040965870Flushing 1588230740/table: creating writer at 1732040965882 (+12 ms)Flushing 1588230740/table: appending metadata at 1732040965896 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732040965896Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@381341a7: reopening flushed file at 1732040965909 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@483a131c: reopening flushed file at 1732040965915 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58bc2371: reopening flushed file at 1732040965921 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 109ms, sequenceid=21, compaction requested=false at 1732040965928 (+7 ms)Writing region close event to WAL at 1732040965929 (+1 ms)Running coprocessor post-close hooks at 1732040965933 (+4 ms)Closed at 1732040965933 2024-11-19T18:29:25,933 DEBUG [RS_CLOSE_META-regionserver/30db5f576be8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T18:29:25,936 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/30db5f576be8,38055,1732040928383 already deleted, retry=false 2024-11-19T18:29:25,936 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 30db5f576be8,38055,1732040928383 expired; onlineServers=1 2024-11-19T18:29:26,019 INFO [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(976): stopping server 30db5f576be8,40263,1732040928486; all regions closed. 2024-11-19T18:29:26,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741836_1012 (size=8899) 2024-11-19T18:29:26,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741836_1012 (size=8899) 2024-11-19T18:29:26,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741836_1012 (size=8899) 2024-11-19T18:29:26,024 DEBUG [RS:1;30db5f576be8:40263 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/oldWALs 2024-11-19T18:29:26,024 INFO [RS:1;30db5f576be8:40263 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 30db5f576be8%2C40263%2C1732040928486.meta:.meta(num 1732040930492) 2024-11-19T18:29:26,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741834_1010 (size=928) 2024-11-19T18:29:26,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741834_1010 (size=928) 2024-11-19T18:29:26,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741834_1010 (size=928) 2024-11-19T18:29:26,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T18:29:26,028 INFO [RS:0;30db5f576be8:38055 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T18:29:26,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38055-0x101317d11230001, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T18:29:26,028 INFO [RS:0;30db5f576be8:38055 {}] regionserver.HRegionServer(1031): Exiting; stopping=30db5f576be8,38055,1732040928383; zookeeper connection closed. 2024-11-19T18:29:26,029 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5e622f02 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5e622f02 2024-11-19T18:29:26,034 DEBUG [RS:1;30db5f576be8:40263 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/oldWALs 2024-11-19T18:29:26,034 INFO [RS:1;30db5f576be8:40263 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 30db5f576be8%2C40263%2C1732040928486:(num 1732040929904) 2024-11-19T18:29:26,034 DEBUG [RS:1;30db5f576be8:40263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T18:29:26,034 INFO [RS:1;30db5f576be8:40263 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T18:29:26,034 INFO [RS:1;30db5f576be8:40263 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T18:29:26,034 INFO [RS:1;30db5f576be8:40263 {}] hbase.ChoreService(370): Chore service for: regionserver/30db5f576be8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T18:29:26,034 INFO [RS:1;30db5f576be8:40263 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T18:29:26,034 INFO [regionserver/30db5f576be8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T18:29:26,034 INFO [RS:1;30db5f576be8:40263 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40263 2024-11-19T18:29:26,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T18:29:26,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/30db5f576be8,40263,1732040928486 2024-11-19T18:29:26,036 INFO [RS:1;30db5f576be8:40263 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T18:29:26,038 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [30db5f576be8,40263,1732040928486] 2024-11-19T18:29:26,039 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/30db5f576be8,40263,1732040928486 already deleted, retry=false 2024-11-19T18:29:26,039 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 30db5f576be8,40263,1732040928486 expired; onlineServers=0 2024-11-19T18:29:26,039 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '30db5f576be8,42887,1732040927686' ***** 2024-11-19T18:29:26,039 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T18:29:26,039 INFO [M:0;30db5f576be8:42887 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T18:29:26,039 INFO [M:0;30db5f576be8:42887 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T18:29:26,039 DEBUG [M:0;30db5f576be8:42887 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T18:29:26,040 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T18:29:26,040 DEBUG [M:0;30db5f576be8:42887 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T18:29:26,040 DEBUG [master/30db5f576be8:0:becomeActiveMaster-HFileCleaner.small.0-1732040929565 {}] cleaner.HFileCleaner(306): Exit Thread[master/30db5f576be8:0:becomeActiveMaster-HFileCleaner.small.0-1732040929565,5,FailOnTimeoutGroup] 2024-11-19T18:29:26,040 DEBUG [master/30db5f576be8:0:becomeActiveMaster-HFileCleaner.large.0-1732040929557 {}] cleaner.HFileCleaner(306): Exit Thread[master/30db5f576be8:0:becomeActiveMaster-HFileCleaner.large.0-1732040929557,5,FailOnTimeoutGroup] 2024-11-19T18:29:26,040 INFO [M:0;30db5f576be8:42887 {}] hbase.ChoreService(370): Chore service for: master/30db5f576be8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T18:29:26,040 INFO [M:0;30db5f576be8:42887 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T18:29:26,040 DEBUG [M:0;30db5f576be8:42887 {}] master.HMaster(1795): Stopping service threads 2024-11-19T18:29:26,040 INFO [M:0;30db5f576be8:42887 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T18:29:26,040 INFO [M:0;30db5f576be8:42887 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T18:29:26,041 INFO [M:0;30db5f576be8:42887 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T18:29:26,041 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T18:29:26,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T18:29:26,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T18:29:26,042 DEBUG [M:0;30db5f576be8:42887 {}] zookeeper.ZKUtil(347): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T18:29:26,042 WARN [M:0;30db5f576be8:42887 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T18:29:26,043 INFO [M:0;30db5f576be8:42887 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/.lastflushedseqids 2024-11-19T18:29:26,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741941_1121 (size=138) 2024-11-19T18:29:26,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741941_1121 (size=138) 2024-11-19T18:29:26,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741941_1121 (size=138) 2024-11-19T18:29:26,059 INFO [M:0;30db5f576be8:42887 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T18:29:26,060 INFO [M:0;30db5f576be8:42887 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T18:29:26,060 DEBUG [M:0;30db5f576be8:42887 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T18:29:26,060 INFO [M:0;30db5f576be8:42887 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T18:29:26,060 DEBUG [M:0;30db5f576be8:42887 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T18:29:26,060 DEBUG [M:0;30db5f576be8:42887 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T18:29:26,060 DEBUG [M:0;30db5f576be8:42887 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T18:29:26,060 INFO [M:0;30db5f576be8:42887 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-11-19T18:29:26,076 DEBUG [M:0;30db5f576be8:42887 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70e6fe65782b41a3b50c6cc900c674a7 is 82, key is hbase:meta,,1/info:regioninfo/1732040930582/Put/seqid=0 2024-11-19T18:29:26,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741942_1122 (size=5672) 2024-11-19T18:29:26,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741942_1122 (size=5672) 2024-11-19T18:29:26,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741942_1122 (size=5672) 2024-11-19T18:29:26,083 INFO [M:0;30db5f576be8:42887 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70e6fe65782b41a3b50c6cc900c674a7 2024-11-19T18:29:26,104 DEBUG [M:0;30db5f576be8:42887 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bdfeb7d97b9a4c349e02fea286d960d4 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732040943095/Put/seqid=0 2024-11-19T18:29:26,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741943_1123 (size=7755) 2024-11-19T18:29:26,114 INFO [M:0;30db5f576be8:42887 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bdfeb7d97b9a4c349e02fea286d960d4 2024-11-19T18:29:26,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741943_1123 (size=7755) 2024-11-19T18:29:26,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741943_1123 (size=7755) 2024-11-19T18:29:26,119 INFO [M:0;30db5f576be8:42887 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bdfeb7d97b9a4c349e02fea286d960d4 2024-11-19T18:29:26,138 INFO [RS:1;30db5f576be8:40263 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T18:29:26,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T18:29:26,138 INFO [RS:1;30db5f576be8:40263 {}] regionserver.HRegionServer(1031): Exiting; stopping=30db5f576be8,40263,1732040928486; zookeeper connection closed. 2024-11-19T18:29:26,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40263-0x101317d11230002, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T18:29:26,139 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@977f096 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@977f096 2024-11-19T18:29:26,139 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-19T18:29:26,140 DEBUG [M:0;30db5f576be8:42887 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bb66f6d201814d0e950a92eeeb974ae5 is 69, key is 30db5f576be8,38055,1732040928383/rs:state/1732040929601/Put/seqid=0 2024-11-19T18:29:26,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741944_1124 (size=5440) 2024-11-19T18:29:26,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741944_1124 (size=5440) 2024-11-19T18:29:26,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741944_1124 (size=5440) 2024-11-19T18:29:26,151 INFO [M:0;30db5f576be8:42887 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bb66f6d201814d0e950a92eeeb974ae5 2024-11-19T18:29:26,156 INFO [M:0;30db5f576be8:42887 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bb66f6d201814d0e950a92eeeb974ae5 2024-11-19T18:29:26,157 DEBUG [M:0;30db5f576be8:42887 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70e6fe65782b41a3b50c6cc900c674a7 as hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/70e6fe65782b41a3b50c6cc900c674a7 2024-11-19T18:29:26,163 INFO [M:0;30db5f576be8:42887 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/70e6fe65782b41a3b50c6cc900c674a7, entries=8, sequenceid=168, filesize=5.5 K 2024-11-19T18:29:26,163 DEBUG [M:0;30db5f576be8:42887 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bdfeb7d97b9a4c349e02fea286d960d4 as hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bdfeb7d97b9a4c349e02fea286d960d4 2024-11-19T18:29:26,169 INFO [M:0;30db5f576be8:42887 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bdfeb7d97b9a4c349e02fea286d960d4 2024-11-19T18:29:26,169 INFO [M:0;30db5f576be8:42887 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bdfeb7d97b9a4c349e02fea286d960d4, entries=17, sequenceid=168, filesize=7.6 K 2024-11-19T18:29:26,170 DEBUG [M:0;30db5f576be8:42887 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bb66f6d201814d0e950a92eeeb974ae5 as hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bb66f6d201814d0e950a92eeeb974ae5 2024-11-19T18:29:26,175 INFO [M:0;30db5f576be8:42887 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bb66f6d201814d0e950a92eeeb974ae5 2024-11-19T18:29:26,175 INFO [M:0;30db5f576be8:42887 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42267/user/jenkins/test-data/4289f340-8c5d-17ef-52c5-fd5da64d3e01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bb66f6d201814d0e950a92eeeb974ae5, entries=3, sequenceid=168, filesize=5.3 K 2024-11-19T18:29:26,176 INFO [M:0;30db5f576be8:42887 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 116ms, sequenceid=168, compaction requested=false 2024-11-19T18:29:26,179 INFO [M:0;30db5f576be8:42887 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T18:29:26,179 DEBUG [M:0;30db5f576be8:42887 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732040966060Disabling compacts and flushes for region at 1732040966060Disabling writes for close at 1732040966060Obtaining lock to block concurrent updates at 1732040966060Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732040966060Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1732040966060Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732040966061 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732040966061Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732040966076 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732040966076Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732040966088 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732040966103 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732040966103Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732040966119 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732040966138 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732040966138Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@136ea7c9: reopening flushed file at 1732040966156 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@304effca: reopening flushed file at 1732040966163 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e5a8c2: reopening flushed file at 1732040966169 (+6 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 116ms, sequenceid=168, compaction requested=false at 1732040966176 (+7 ms)Writing region close event to WAL at 1732040966179 (+3 ms)Closed at 1732040966179 2024-11-19T18:29:26,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33471 is added to blk_1073741830_1006 (size=81431) 2024-11-19T18:29:26,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41941 is added to blk_1073741830_1006 (size=81431) 2024-11-19T18:29:26,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33841 is added to blk_1073741830_1006 (size=81431) 2024-11-19T18:29:26,184 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T18:29:26,184 INFO [M:0;30db5f576be8:42887 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T18:29:26,184 INFO [M:0;30db5f576be8:42887 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42887 2024-11-19T18:29:26,185 INFO [M:0;30db5f576be8:42887 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T18:29:26,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T18:29:26,287 INFO [M:0;30db5f576be8:42887 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T18:29:26,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42887-0x101317d11230000, quorum=127.0.0.1:52143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T18:29:26,294 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946473 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946473 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1454744767_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732040946147/wal.1732040946473 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1454744767_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-19T18:29:26,296 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732040938229/wal.1732040938342 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:26,296 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732040961204/wal.1732040961396 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:26,299 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040955927 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040955927 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1454744767_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732040946647/wal.1732040955927 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1454744767_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-19T18:29:26,300 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732040931851/wal.1732040931928 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:26,302 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956536 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956536 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1454744767_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732040956116/wal.1732040956536 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1454744767_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-19T18:29:26,302 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732040956689/wal.1732040956808 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T18:29:26,304 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal.1732040931712 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal.1732040931712 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1454744767_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732040931531/wal.1732040931712 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1454744767_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-19T18:29:26,307 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal.1732040931231 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal.1732040931231 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1454744767_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732040930964/wal.1732040931231 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1454744767_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-19T18:29:26,313 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3217543c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T18:29:26,316 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T18:29:26,316 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T18:29:26,316 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T18:29:26,316 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/hadoop.log.dir/,STOPPED} 2024-11-19T18:29:26,320 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-19T18:29:26,320 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-19T18:29:26,320 WARN [BP-1034186995-172.17.0.2-1732040924630 heartbeating to localhost/127.0.0.1:42267 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-19T18:29:26,320 WARN [BP-1034186995-172.17.0.2-1732040924630 heartbeating to localhost/127.0.0.1:42267 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1034186995-172.17.0.2-1732040924630 (Datanode Uuid 9ab5956c-c134-4391-8dd3-6fd734f8a877) service to localhost/127.0.0.1:42267 2024-11-19T18:29:26,322 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/data/data5/current/BP-1034186995-172.17.0.2-1732040924630 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T18:29:26,322 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/data/data6/current/BP-1034186995-172.17.0.2-1732040924630 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T18:29:26,323 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-19T18:29:26,325 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7e9308b2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T18:29:26,326 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T18:29:26,326 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T18:29:26,326 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T18:29:26,326 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/hadoop.log.dir/,STOPPED} 2024-11-19T18:29:26,328 WARN [BP-1034186995-172.17.0.2-1732040924630 heartbeating to localhost/127.0.0.1:42267 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-19T18:29:26,328 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-19T18:29:26,328 WARN [BP-1034186995-172.17.0.2-1732040924630 heartbeating to localhost/127.0.0.1:42267 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1034186995-172.17.0.2-1732040924630 (Datanode Uuid 58c9e594-4038-46bc-aaf1-0f2d1735a697) service to localhost/127.0.0.1:42267 2024-11-19T18:29:26,328 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-19T18:29:26,328 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/data/data3/current/BP-1034186995-172.17.0.2-1732040924630 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T18:29:26,329 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/data/data4/current/BP-1034186995-172.17.0.2-1732040924630 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T18:29:26,329 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-19T18:29:26,337 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54387323{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T18:29:26,338 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T18:29:26,338 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T18:29:26,338 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T18:29:26,338 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/hadoop.log.dir/,STOPPED} 2024-11-19T18:29:26,340 WARN [BP-1034186995-172.17.0.2-1732040924630 heartbeating to localhost/127.0.0.1:42267 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-19T18:29:26,340 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-19T18:29:26,340 WARN [BP-1034186995-172.17.0.2-1732040924630 heartbeating to localhost/127.0.0.1:42267 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1034186995-172.17.0.2-1732040924630 (Datanode Uuid df847fab-12ca-4b42-956e-a0a03b7ef6d8) service to localhost/127.0.0.1:42267 2024-11-19T18:29:26,340 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-19T18:29:26,341 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/data/data1/current/BP-1034186995-172.17.0.2-1732040924630 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T18:29:26,341 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/cluster_8740c3de-5e95-c7de-a33c-8d576529a9b8/data/data2/current/BP-1034186995-172.17.0.2-1732040924630 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T18:29:26,341 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-19T18:29:26,350 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@398d2d49{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T18:29:26,351 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T18:29:26,351 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T18:29:26,351 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T18:29:26,351 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c23623a7-7e2f-3a99-c391-2e8f59c9c378/hadoop.log.dir/,STOPPED} 2024-11-19T18:29:26,363 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T18:29:26,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down