2024-12-03 02:19:50,651 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-12-03 02:19:50,666 main DEBUG Took 0.013167 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 02:19:50,667 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 02:19:50,667 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 02:19:50,669 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 02:19:50,671 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:19:50,680 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 02:19:50,694 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:19:50,695 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:19:50,696 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:19:50,697 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:19:50,697 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:19:50,698 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:19:50,699 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:19:50,700 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:19:50,700 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:19:50,701 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:19:50,702 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:19:50,702 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:19:50,703 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:19:50,704 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:19:50,704 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:19:50,705 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:19:50,705 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:19:50,706 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:19:50,706 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:19:50,707 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:19:50,707 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:19:50,708 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:19:50,708 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:19:50,709 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:19:50,709 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:19:50,710 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 02:19:50,712 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:19:50,713 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 02:19:50,716 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 02:19:50,717 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 02:19:50,718 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 02:19:50,719 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 02:19:50,730 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 02:19:50,734 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 02:19:50,736 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 02:19:50,736 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 02:19:50,737 main DEBUG createAppenders(={Console}) 2024-12-03 02:19:50,738 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-12-03 02:19:50,738 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-12-03 02:19:50,739 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-12-03 02:19:50,740 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 02:19:50,740 main DEBUG OutputStream closed 2024-12-03 02:19:50,740 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 02:19:50,741 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 02:19:50,741 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-12-03 02:19:50,822 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 02:19:50,824 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 02:19:50,825 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 02:19:50,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 02:19:50,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 02:19:50,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 02:19:50,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 02:19:50,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 02:19:50,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 02:19:50,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 02:19:50,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 02:19:50,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 02:19:50,829 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 02:19:50,829 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 02:19:50,829 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 02:19:50,830 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 02:19:50,830 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 02:19:50,831 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 02:19:50,833 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 02:19:50,833 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-12-03 02:19:50,833 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 02:19:50,834 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-12-03T02:19:51,065 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b 2024-12-03 02:19:51,068 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 02:19:51,069 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T02:19:51,079 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-03T02:19:51,106 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T02:19:51,148 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T02:19:51,148 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T02:19:51,160 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T02:19:51,178 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78, deleteOnExit=true 2024-12-03T02:19:51,178 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T02:19:51,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/test.cache.data in system properties and HBase conf 2024-12-03T02:19:51,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T02:19:51,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/hadoop.log.dir in system properties and HBase conf 2024-12-03T02:19:51,181 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T02:19:51,181 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T02:19:51,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T02:19:51,276 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T02:19:51,364 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T02:19:51,367 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T02:19:51,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T02:19:51,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T02:19:51,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T02:19:51,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T02:19:51,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T02:19:51,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T02:19:51,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T02:19:51,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T02:19:51,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/nfs.dump.dir in system properties and HBase conf 2024-12-03T02:19:51,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/java.io.tmpdir in system properties and HBase conf 2024-12-03T02:19:51,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T02:19:51,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T02:19:51,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T02:19:52,650 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T02:19:52,713 INFO [Time-limited test {}] log.Log(170): Logging initialized @2995ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T02:19:52,778 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T02:19:52,841 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T02:19:52,861 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T02:19:52,861 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T02:19:52,862 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T02:19:52,874 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T02:19:52,877 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/hadoop.log.dir/,AVAILABLE} 2024-12-03T02:19:52,879 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T02:19:53,041 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@398d2d49{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/java.io.tmpdir/jetty-localhost-33873-hadoop-hdfs-3_4_1-tests_jar-_-any-13284577487962606057/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T02:19:53,047 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:33873} 2024-12-03T02:19:53,047 INFO [Time-limited test {}] server.Server(415): Started @3329ms 2024-12-03T02:19:53,611 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T02:19:53,617 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T02:19:53,618 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T02:19:53,618 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T02:19:53,618 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T02:19:53,619 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ce178fc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/hadoop.log.dir/,AVAILABLE} 2024-12-03T02:19:53,619 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32ec6e08{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T02:19:53,722 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2ad3efb4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/java.io.tmpdir/jetty-localhost-45987-hadoop-hdfs-3_4_1-tests_jar-_-any-1858561418760804028/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T02:19:53,723 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3b68aeb4{HTTP/1.1, (http/1.1)}{localhost:45987} 2024-12-03T02:19:53,723 INFO [Time-limited test {}] server.Server(415): Started @4004ms 2024-12-03T02:19:53,769 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T02:19:53,931 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T02:19:53,935 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T02:19:53,941 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T02:19:53,941 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T02:19:53,941 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T02:19:53,944 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@918952d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/hadoop.log.dir/,AVAILABLE} 2024-12-03T02:19:53,944 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ca59792{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T02:19:54,063 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7742c079{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/java.io.tmpdir/jetty-localhost-37155-hadoop-hdfs-3_4_1-tests_jar-_-any-7504916835475999349/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T02:19:54,063 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7dd93ad{HTTP/1.1, (http/1.1)}{localhost:37155} 2024-12-03T02:19:54,064 INFO [Time-limited test {}] server.Server(415): Started @4345ms 2024-12-03T02:19:54,065 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T02:19:54,100 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T02:19:54,105 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T02:19:54,107 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T02:19:54,107 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T02:19:54,107 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T02:19:54,108 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b2b98f9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/hadoop.log.dir/,AVAILABLE} 2024-12-03T02:19:54,109 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70c89de7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T02:19:54,208 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@211150ab{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/java.io.tmpdir/jetty-localhost-38247-hadoop-hdfs-3_4_1-tests_jar-_-any-5215532414536251497/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T02:19:54,208 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3babed56{HTTP/1.1, (http/1.1)}{localhost:38247} 2024-12-03T02:19:54,209 INFO [Time-limited test {}] server.Server(415): Started @4490ms 2024-12-03T02:19:54,211 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T02:19:55,620 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data1/current/BP-537057715-172.17.0.3-1733192391930/current, will proceed with Du for space computation calculation, 2024-12-03T02:19:55,620 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data2/current/BP-537057715-172.17.0.3-1733192391930/current, will proceed with Du for space computation calculation, 2024-12-03T02:19:55,620 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data3/current/BP-537057715-172.17.0.3-1733192391930/current, will proceed with Du for space computation calculation, 2024-12-03T02:19:55,620 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data4/current/BP-537057715-172.17.0.3-1733192391930/current, will proceed with Du for space computation calculation, 2024-12-03T02:19:55,671 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T02:19:55,671 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T02:19:55,718 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd1c931b0c771cb7b with lease ID 0xd415a928e7f577c6: Processing first storage report for DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f from datanode DatanodeRegistration(127.0.0.1:35125, datanodeUuid=8ba6be36-95ab-4a1f-9b4a-06a3af00259d, infoPort=36675, infoSecurePort=0, ipcPort=41795, storageInfo=lv=-57;cid=testClusterID;nsid=929572102;c=1733192391930) 2024-12-03T02:19:55,719 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd1c931b0c771cb7b with lease ID 0xd415a928e7f577c6: from storage DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f node DatanodeRegistration(127.0.0.1:35125, datanodeUuid=8ba6be36-95ab-4a1f-9b4a-06a3af00259d, infoPort=36675, infoSecurePort=0, ipcPort=41795, storageInfo=lv=-57;cid=testClusterID;nsid=929572102;c=1733192391930), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-03T02:19:55,720 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa4c6847add612477 with lease ID 0xd415a928e7f577c5: Processing first storage report for DS-01018773-b73a-481d-8884-49ff5e2559e9 from datanode DatanodeRegistration(127.0.0.1:43037, datanodeUuid=05ad8527-09ed-4ccd-9721-cc32ed0ed941, infoPort=43847, infoSecurePort=0, ipcPort=34205, storageInfo=lv=-57;cid=testClusterID;nsid=929572102;c=1733192391930) 2024-12-03T02:19:55,720 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa4c6847add612477 with lease ID 0xd415a928e7f577c5: from storage DS-01018773-b73a-481d-8884-49ff5e2559e9 node DatanodeRegistration(127.0.0.1:43037, datanodeUuid=05ad8527-09ed-4ccd-9721-cc32ed0ed941, infoPort=43847, infoSecurePort=0, ipcPort=34205, storageInfo=lv=-57;cid=testClusterID;nsid=929572102;c=1733192391930), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T02:19:55,720 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd1c931b0c771cb7b with lease ID 0xd415a928e7f577c6: Processing first storage report for DS-db43c4a0-3bd5-4cde-a2cf-d4edaed3c3d8 from datanode DatanodeRegistration(127.0.0.1:35125, datanodeUuid=8ba6be36-95ab-4a1f-9b4a-06a3af00259d, infoPort=36675, infoSecurePort=0, ipcPort=41795, storageInfo=lv=-57;cid=testClusterID;nsid=929572102;c=1733192391930) 2024-12-03T02:19:55,720 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd1c931b0c771cb7b with lease ID 0xd415a928e7f577c6: from storage DS-db43c4a0-3bd5-4cde-a2cf-d4edaed3c3d8 node DatanodeRegistration(127.0.0.1:35125, datanodeUuid=8ba6be36-95ab-4a1f-9b4a-06a3af00259d, infoPort=36675, infoSecurePort=0, ipcPort=41795, storageInfo=lv=-57;cid=testClusterID;nsid=929572102;c=1733192391930), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T02:19:55,721 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa4c6847add612477 with lease ID 0xd415a928e7f577c5: Processing first storage report for DS-02497c29-bc2d-47bb-87dd-265bd72ee906 from datanode DatanodeRegistration(127.0.0.1:43037, datanodeUuid=05ad8527-09ed-4ccd-9721-cc32ed0ed941, infoPort=43847, infoSecurePort=0, ipcPort=34205, storageInfo=lv=-57;cid=testClusterID;nsid=929572102;c=1733192391930) 2024-12-03T02:19:55,721 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa4c6847add612477 with lease ID 0xd415a928e7f577c5: from storage DS-02497c29-bc2d-47bb-87dd-265bd72ee906 node DatanodeRegistration(127.0.0.1:43037, datanodeUuid=05ad8527-09ed-4ccd-9721-cc32ed0ed941, infoPort=43847, infoSecurePort=0, ipcPort=34205, storageInfo=lv=-57;cid=testClusterID;nsid=929572102;c=1733192391930), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T02:19:55,747 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data6/current/BP-537057715-172.17.0.3-1733192391930/current, will proceed with Du for space computation calculation, 2024-12-03T02:19:55,747 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data5/current/BP-537057715-172.17.0.3-1733192391930/current, will proceed with Du for space computation calculation, 2024-12-03T02:19:55,764 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T02:19:55,768 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x25dfb300b4003aec with lease ID 0xd415a928e7f577c7: Processing first storage report for DS-51edcd96-2d93-432f-b830-e47c1264fe0a from datanode DatanodeRegistration(127.0.0.1:36885, datanodeUuid=d5b007e8-31d1-4b23-91ab-1d4bdb9ba5b2, infoPort=39215, infoSecurePort=0, ipcPort=45513, storageInfo=lv=-57;cid=testClusterID;nsid=929572102;c=1733192391930) 2024-12-03T02:19:55,768 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x25dfb300b4003aec with lease ID 0xd415a928e7f577c7: from storage DS-51edcd96-2d93-432f-b830-e47c1264fe0a node DatanodeRegistration(127.0.0.1:36885, datanodeUuid=d5b007e8-31d1-4b23-91ab-1d4bdb9ba5b2, infoPort=39215, infoSecurePort=0, ipcPort=45513, storageInfo=lv=-57;cid=testClusterID;nsid=929572102;c=1733192391930), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T02:19:55,768 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x25dfb300b4003aec with lease ID 0xd415a928e7f577c7: Processing first storage report for DS-c1648c02-9e0f-4d34-a9a6-92ef043e843e from datanode DatanodeRegistration(127.0.0.1:36885, datanodeUuid=d5b007e8-31d1-4b23-91ab-1d4bdb9ba5b2, infoPort=39215, infoSecurePort=0, ipcPort=45513, storageInfo=lv=-57;cid=testClusterID;nsid=929572102;c=1733192391930) 2024-12-03T02:19:55,768 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x25dfb300b4003aec with lease ID 0xd415a928e7f577c7: from storage DS-c1648c02-9e0f-4d34-a9a6-92ef043e843e node DatanodeRegistration(127.0.0.1:36885, datanodeUuid=d5b007e8-31d1-4b23-91ab-1d4bdb9ba5b2, infoPort=39215, infoSecurePort=0, ipcPort=45513, storageInfo=lv=-57;cid=testClusterID;nsid=929572102;c=1733192391930), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T02:19:55,874 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b 2024-12-03T02:19:55,957 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/zookeeper_0, clientPort=57355, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T02:19:55,968 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57355 2024-12-03T02:19:55,979 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:19:55,982 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:19:56,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741825_1001 (size=7) 2024-12-03T02:19:56,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741825_1001 (size=7) 2024-12-03T02:19:56,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741825_1001 (size=7) 2024-12-03T02:19:56,585 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c with version=8 2024-12-03T02:19:56,585 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/hbase-staging 2024-12-03T02:19:56,860 INFO [Time-limited test {}] client.ConnectionUtils(128): master/64035cbb0b46:0 server-side Connection retries=45 2024-12-03T02:19:56,868 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T02:19:56,869 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T02:19:56,874 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T02:19:56,874 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T02:19:56,874 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T02:19:56,996 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T02:19:57,046 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T02:19:57,055 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T02:19:57,059 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T02:19:57,093 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 42756 (auto-detected) 2024-12-03T02:19:57,094 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-03T02:19:57,117 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33741 2024-12-03T02:19:57,135 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33741 connecting to ZooKeeper ensemble=127.0.0.1:57355 2024-12-03T02:19:57,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:337410x0, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T02:19:57,269 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33741-0x1019a46de8c0000 connected 2024-12-03T02:19:57,358 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:19:57,363 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:19:57,372 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T02:19:57,376 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c, hbase.cluster.distributed=false 2024-12-03T02:19:57,398 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T02:19:57,403 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33741 2024-12-03T02:19:57,403 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33741 2024-12-03T02:19:57,405 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33741 2024-12-03T02:19:57,406 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33741 2024-12-03T02:19:57,407 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33741 2024-12-03T02:19:57,507 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/64035cbb0b46:0 server-side Connection retries=45 2024-12-03T02:19:57,510 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T02:19:57,510 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T02:19:57,511 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T02:19:57,511 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T02:19:57,511 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T02:19:57,514 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T02:19:57,517 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T02:19:57,518 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34333 2024-12-03T02:19:57,520 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34333 connecting to ZooKeeper ensemble=127.0.0.1:57355 2024-12-03T02:19:57,521 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:19:57,524 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:19:57,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:343330x0, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T02:19:57,540 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:343330x0, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T02:19:57,540 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34333-0x1019a46de8c0001 connected 2024-12-03T02:19:57,544 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T02:19:57,552 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T02:19:57,554 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34333-0x1019a46de8c0001, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T02:19:57,560 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34333-0x1019a46de8c0001, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T02:19:57,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34333 2024-12-03T02:19:57,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34333 2024-12-03T02:19:57,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34333 2024-12-03T02:19:57,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34333 2024-12-03T02:19:57,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34333 2024-12-03T02:19:57,577 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/64035cbb0b46:0 server-side Connection retries=45 2024-12-03T02:19:57,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T02:19:57,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T02:19:57,577 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T02:19:57,578 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T02:19:57,578 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T02:19:57,578 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T02:19:57,578 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T02:19:57,579 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44467 2024-12-03T02:19:57,580 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44467 connecting to ZooKeeper ensemble=127.0.0.1:57355 2024-12-03T02:19:57,581 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:19:57,584 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:19:57,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:444670x0, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T02:19:57,598 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:444670x0, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T02:19:57,598 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44467-0x1019a46de8c0002 connected 2024-12-03T02:19:57,599 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T02:19:57,600 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T02:19:57,602 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T02:19:57,603 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T02:19:57,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44467 2024-12-03T02:19:57,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44467 2024-12-03T02:19:57,605 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44467 2024-12-03T02:19:57,605 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44467 2024-12-03T02:19:57,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44467 2024-12-03T02:19:57,620 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/64035cbb0b46:0 server-side Connection retries=45 2024-12-03T02:19:57,620 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T02:19:57,620 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T02:19:57,620 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T02:19:57,620 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T02:19:57,621 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T02:19:57,621 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T02:19:57,621 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T02:19:57,622 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39333 2024-12-03T02:19:57,623 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39333 connecting to ZooKeeper ensemble=127.0.0.1:57355 2024-12-03T02:19:57,624 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:19:57,626 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:19:57,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:393330x0, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T02:19:57,640 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:393330x0, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T02:19:57,640 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39333-0x1019a46de8c0003 connected 2024-12-03T02:19:57,640 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T02:19:57,641 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T02:19:57,642 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T02:19:57,644 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T02:19:57,645 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39333 2024-12-03T02:19:57,645 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39333 2024-12-03T02:19:57,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39333 2024-12-03T02:19:57,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39333 2024-12-03T02:19:57,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39333 2024-12-03T02:19:57,658 DEBUG [M:0;64035cbb0b46:33741 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;64035cbb0b46:33741 2024-12-03T02:19:57,659 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/64035cbb0b46,33741,1733192396696 2024-12-03T02:19:57,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T02:19:57,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T02:19:57,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34333-0x1019a46de8c0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T02:19:57,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T02:19:57,679 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/64035cbb0b46,33741,1733192396696 2024-12-03T02:19:57,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T02:19:57,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34333-0x1019a46de8c0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T02:19:57,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T02:19:57,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:57,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:57,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34333-0x1019a46de8c0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:57,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:57,715 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T02:19:57,717 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/64035cbb0b46,33741,1733192396696 from backup master directory 2024-12-03T02:19:57,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T02:19:57,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/64035cbb0b46,33741,1733192396696 2024-12-03T02:19:57,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34333-0x1019a46de8c0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T02:19:57,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T02:19:57,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T02:19:57,724 WARN [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T02:19:57,724 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=64035cbb0b46,33741,1733192396696 2024-12-03T02:19:57,726 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T02:19:57,731 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T02:19:57,795 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/hbase.id] with ID: 1be2a799-8e6e-40d2-897e-f78b191efd08 2024-12-03T02:19:57,796 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/.tmp/hbase.id 2024-12-03T02:19:57,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741826_1002 (size=42) 2024-12-03T02:19:57,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741826_1002 (size=42) 2024-12-03T02:19:57,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741826_1002 (size=42) 2024-12-03T02:19:57,818 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/.tmp/hbase.id]:[hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/hbase.id] 2024-12-03T02:19:57,871 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:19:57,875 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T02:19:57,895 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-12-03T02:19:57,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:57,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:57,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34333-0x1019a46de8c0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:57,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:57,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741827_1003 (size=196) 2024-12-03T02:19:57,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741827_1003 (size=196) 2024-12-03T02:19:57,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741827_1003 (size=196) 2024-12-03T02:19:57,955 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T02:19:57,956 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T02:19:57,961 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:19:57,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741828_1004 (size=1189) 2024-12-03T02:19:57,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741828_1004 (size=1189) 2024-12-03T02:19:57,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741828_1004 (size=1189) 2024-12-03T02:19:58,009 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store 2024-12-03T02:19:58,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741829_1005 (size=34) 2024-12-03T02:19:58,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741829_1005 (size=34) 2024-12-03T02:19:58,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741829_1005 (size=34) 2024-12-03T02:19:58,035 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T02:19:58,038 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:19:58,039 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T02:19:58,039 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:19:58,039 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:19:58,041 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms 2024-12-03T02:19:58,041 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:19:58,041 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:19:58,042 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733192398039Disabling compacts and flushes for region at 1733192398039Disabling writes for close at 1733192398041 (+2 ms)Writing region close event to WAL at 1733192398041Closed at 1733192398041 2024-12-03T02:19:58,044 WARN [master/64035cbb0b46:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/.initializing 2024-12-03T02:19:58,044 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/WALs/64035cbb0b46,33741,1733192396696 2024-12-03T02:19:58,052 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:19:58,069 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=64035cbb0b46%2C33741%2C1733192396696, suffix=, logDir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/WALs/64035cbb0b46,33741,1733192396696, archiveDir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/oldWALs, maxLogs=10 2024-12-03T02:19:58,100 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/WALs/64035cbb0b46,33741,1733192396696/64035cbb0b46%2C33741%2C1733192396696.1733192398075, exclude list is [], retry=0 2024-12-03T02:19:58,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:19:58,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:19:58,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:19:58,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:19:58,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T02:19:58,161 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/WALs/64035cbb0b46,33741,1733192396696/64035cbb0b46%2C33741%2C1733192396696.1733192398075 2024-12-03T02:19:58,162 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:39215:39215)] 2024-12-03T02:19:58,162 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:19:58,163 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:19:58,165 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:19:58,166 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:19:58,203 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:19:58,234 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T02:19:58,238 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:19:58,241 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:19:58,241 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:19:58,245 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T02:19:58,245 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:19:58,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:19:58,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:19:58,249 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T02:19:58,249 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:19:58,250 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:19:58,251 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:19:58,254 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T02:19:58,254 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:19:58,255 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:19:58,256 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:19:58,260 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:19:58,261 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:19:58,266 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:19:58,267 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:19:58,270 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T02:19:58,274 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:19:58,279 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:19:58,280 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68254857, jitterRate=0.017076626420021057}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T02:19:58,286 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733192398178Initializing all the Stores at 1733192398180 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733192398180Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192398181 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192398181Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192398181Cleaning up temporary data from old regions at 1733192398267 (+86 ms)Region opened successfully at 1733192398286 (+19 ms) 2024-12-03T02:19:58,287 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T02:19:58,319 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ce23cc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=64035cbb0b46/172.17.0.3:0 2024-12-03T02:19:58,354 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T02:19:58,367 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T02:19:58,367 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T02:19:58,370 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T02:19:58,371 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T02:19:58,376 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-03T02:19:58,376 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T02:19:58,399 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T02:19:58,407 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T02:19:58,414 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T02:19:58,417 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T02:19:58,419 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T02:19:58,430 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T02:19:58,433 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T02:19:58,437 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T02:19:58,447 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T02:19:58,449 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T02:19:58,455 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T02:19:58,475 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T02:19:58,480 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T02:19:58,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T02:19:58,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T02:19:58,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T02:19:58,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34333-0x1019a46de8c0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T02:19:58,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34333-0x1019a46de8c0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:58,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:58,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:58,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:58,494 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=64035cbb0b46,33741,1733192396696, sessionid=0x1019a46de8c0000, setting cluster-up flag (Was=false) 2024-12-03T02:19:58,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:58,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:58,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:58,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34333-0x1019a46de8c0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:58,539 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T02:19:58,541 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=64035cbb0b46,33741,1733192396696 2024-12-03T02:19:58,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:58,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:58,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:58,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34333-0x1019a46de8c0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:58,622 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T02:19:58,625 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=64035cbb0b46,33741,1733192396696 2024-12-03T02:19:58,633 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T02:19:58,650 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(746): ClusterId : 1be2a799-8e6e-40d2-897e-f78b191efd08 2024-12-03T02:19:58,650 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(746): ClusterId : 1be2a799-8e6e-40d2-897e-f78b191efd08 2024-12-03T02:19:58,650 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(746): ClusterId : 1be2a799-8e6e-40d2-897e-f78b191efd08 2024-12-03T02:19:58,652 DEBUG [RS:0;64035cbb0b46:34333 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T02:19:58,652 DEBUG [RS:2;64035cbb0b46:39333 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T02:19:58,652 DEBUG [RS:1;64035cbb0b46:44467 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T02:19:58,667 DEBUG [RS:0;64035cbb0b46:34333 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T02:19:58,667 DEBUG [RS:2;64035cbb0b46:39333 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T02:19:58,667 DEBUG [RS:1;64035cbb0b46:44467 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T02:19:58,667 DEBUG [RS:1;64035cbb0b46:44467 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T02:19:58,667 DEBUG [RS:0;64035cbb0b46:34333 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T02:19:58,667 DEBUG [RS:2;64035cbb0b46:39333 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T02:19:58,682 DEBUG [RS:0;64035cbb0b46:34333 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T02:19:58,682 DEBUG [RS:2;64035cbb0b46:39333 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T02:19:58,682 DEBUG [RS:1;64035cbb0b46:44467 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T02:19:58,683 DEBUG [RS:1;64035cbb0b46:44467 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@550948a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=64035cbb0b46/172.17.0.3:0 2024-12-03T02:19:58,683 DEBUG [RS:0;64035cbb0b46:34333 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2062eeb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=64035cbb0b46/172.17.0.3:0 2024-12-03T02:19:58,683 DEBUG [RS:2;64035cbb0b46:39333 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f07e124, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=64035cbb0b46/172.17.0.3:0 2024-12-03T02:19:58,702 DEBUG [RS:0;64035cbb0b46:34333 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;64035cbb0b46:34333 2024-12-03T02:19:58,703 DEBUG [RS:1;64035cbb0b46:44467 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;64035cbb0b46:44467 2024-12-03T02:19:58,705 DEBUG [RS:2;64035cbb0b46:39333 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;64035cbb0b46:39333 2024-12-03T02:19:58,706 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T02:19:58,706 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T02:19:58,706 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T02:19:58,706 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T02:19:58,706 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T02:19:58,706 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T02:19:58,707 DEBUG [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T02:19:58,707 DEBUG [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T02:19:58,707 DEBUG [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T02:19:58,710 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(2659): reportForDuty to master=64035cbb0b46,33741,1733192396696 with port=39333, startcode=1733192397619 2024-12-03T02:19:58,710 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(2659): reportForDuty to master=64035cbb0b46,33741,1733192396696 with port=34333, startcode=1733192397467 2024-12-03T02:19:58,710 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(2659): reportForDuty to master=64035cbb0b46,33741,1733192396696 with port=44467, startcode=1733192397576 2024-12-03T02:19:58,717 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T02:19:58,723 DEBUG [RS:1;64035cbb0b46:44467 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T02:19:58,723 DEBUG [RS:0;64035cbb0b46:34333 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T02:19:58,723 DEBUG [RS:2;64035cbb0b46:39333 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T02:19:58,731 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T02:19:58,739 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T02:19:58,746 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 64035cbb0b46,33741,1733192396696 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T02:19:58,765 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47921, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T02:19:58,765 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33533, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T02:19:58,766 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/64035cbb0b46:0, corePoolSize=5, maxPoolSize=5 2024-12-03T02:19:58,766 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/64035cbb0b46:0, corePoolSize=5, maxPoolSize=5 2024-12-03T02:19:58,766 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45967, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T02:19:58,766 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/64035cbb0b46:0, corePoolSize=5, maxPoolSize=5 2024-12-03T02:19:58,766 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/64035cbb0b46:0, corePoolSize=5, maxPoolSize=5 2024-12-03T02:19:58,766 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/64035cbb0b46:0, corePoolSize=10, maxPoolSize=10 2024-12-03T02:19:58,767 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:58,767 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/64035cbb0b46:0, corePoolSize=2, maxPoolSize=2 2024-12-03T02:19:58,767 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:58,776 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33741 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:19:58,780 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733192428780 2024-12-03T02:19:58,782 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T02:19:58,782 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33741 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:19:58,783 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T02:19:58,783 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T02:19:58,783 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T02:19:58,783 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33741 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:19:58,789 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T02:19:58,789 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T02:19:58,789 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T02:19:58,790 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T02:19:58,791 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:19:58,791 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T02:19:58,790 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:58,796 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T02:19:58,797 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T02:19:58,798 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T02:19:58,807 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T02:19:58,808 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T02:19:58,810 DEBUG [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-03T02:19:58,810 DEBUG [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-03T02:19:58,810 WARN [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-03T02:19:58,810 WARN [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-03T02:19:58,810 DEBUG [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-03T02:19:58,811 WARN [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-03T02:19:58,811 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.large.0-1733192398810,5,FailOnTimeoutGroup] 2024-12-03T02:19:58,814 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.small.0-1733192398811,5,FailOnTimeoutGroup] 2024-12-03T02:19:58,814 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:58,814 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T02:19:58,816 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:58,816 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:58,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741831_1007 (size=1321) 2024-12-03T02:19:58,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741831_1007 (size=1321) 2024-12-03T02:19:58,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741831_1007 (size=1321) 2024-12-03T02:19:58,868 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T02:19:58,869 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c 2024-12-03T02:19:58,912 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(2659): reportForDuty to master=64035cbb0b46,33741,1733192396696 with port=39333, startcode=1733192397619 2024-12-03T02:19:58,912 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(2659): reportForDuty to master=64035cbb0b46,33741,1733192396696 with port=44467, startcode=1733192397576 2024-12-03T02:19:58,913 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(2659): reportForDuty to master=64035cbb0b46,33741,1733192396696 with port=34333, startcode=1733192397467 2024-12-03T02:19:58,915 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33741 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 64035cbb0b46,44467,1733192397576 2024-12-03T02:19:58,918 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33741 {}] master.ServerManager(517): Registering regionserver=64035cbb0b46,44467,1733192397576 2024-12-03T02:19:58,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741832_1008 (size=32) 2024-12-03T02:19:58,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741832_1008 (size=32) 2024-12-03T02:19:58,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741832_1008 (size=32) 2024-12-03T02:19:58,926 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:19:58,929 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33741 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 64035cbb0b46,34333,1733192397467 2024-12-03T02:19:58,929 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33741 {}] master.ServerManager(517): Registering regionserver=64035cbb0b46,34333,1733192397467 2024-12-03T02:19:58,929 DEBUG [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c 2024-12-03T02:19:58,929 DEBUG [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44855 2024-12-03T02:19:58,930 DEBUG [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T02:19:58,930 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T02:19:58,934 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33741 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 64035cbb0b46,39333,1733192397619 2024-12-03T02:19:58,934 DEBUG [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c 2024-12-03T02:19:58,934 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33741 {}] master.ServerManager(517): Registering regionserver=64035cbb0b46,39333,1733192397619 2024-12-03T02:19:58,934 DEBUG [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44855 2024-12-03T02:19:58,934 DEBUG [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T02:19:58,936 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T02:19:58,936 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:19:58,939 DEBUG [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c 2024-12-03T02:19:58,939 DEBUG [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44855 2024-12-03T02:19:58,939 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:19:58,939 DEBUG [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T02:19:58,939 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T02:19:58,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T02:19:58,944 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T02:19:58,945 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:19:58,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:19:58,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T02:19:58,951 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T02:19:58,952 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:19:58,953 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:19:58,953 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T02:19:58,956 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T02:19:58,957 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:19:58,958 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:19:58,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T02:19:58,959 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740 2024-12-03T02:19:58,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740 2024-12-03T02:19:58,964 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T02:19:58,964 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T02:19:58,965 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T02:19:58,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T02:19:58,974 DEBUG [RS:1;64035cbb0b46:44467 {}] zookeeper.ZKUtil(111): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/64035cbb0b46,44467,1733192397576 2024-12-03T02:19:58,974 WARN [RS:1;64035cbb0b46:44467 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T02:19:58,974 INFO [RS:1;64035cbb0b46:44467 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:19:58,974 DEBUG [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,44467,1733192397576 2024-12-03T02:19:58,975 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:19:58,976 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64760022, jitterRate=-0.03500047326087952}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T02:19:58,979 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733192398926Initializing all the Stores at 1733192398929 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733192398929Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733192398930 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192398930Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733192398930Cleaning up temporary data from old regions at 1733192398964 (+34 ms)Region opened successfully at 1733192398979 (+15 ms) 2024-12-03T02:19:58,979 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T02:19:58,980 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T02:19:58,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T02:19:58,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T02:19:58,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T02:19:58,981 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T02:19:58,981 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733192398979Disabling compacts and flushes for region at 1733192398979Disabling writes for close at 1733192398980 (+1 ms)Writing region close event to WAL at 1733192398981 (+1 ms)Closed at 1733192398981 2024-12-03T02:19:58,986 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T02:19:58,986 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T02:19:58,989 DEBUG [RS:0;64035cbb0b46:34333 {}] zookeeper.ZKUtil(111): regionserver:34333-0x1019a46de8c0001, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/64035cbb0b46,34333,1733192397467 2024-12-03T02:19:58,990 WARN [RS:0;64035cbb0b46:34333 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T02:19:58,990 INFO [RS:0;64035cbb0b46:34333 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:19:58,990 DEBUG [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467 2024-12-03T02:19:58,992 DEBUG [RS:2;64035cbb0b46:39333 {}] zookeeper.ZKUtil(111): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/64035cbb0b46,39333,1733192397619 2024-12-03T02:19:58,992 WARN [RS:2;64035cbb0b46:39333 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T02:19:58,993 INFO [RS:2;64035cbb0b46:39333 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:19:58,993 DEBUG [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,39333,1733192397619 2024-12-03T02:19:58,993 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [64035cbb0b46,44467,1733192397576] 2024-12-03T02:19:58,994 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [64035cbb0b46,34333,1733192397467] 2024-12-03T02:19:58,994 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [64035cbb0b46,39333,1733192397619] 2024-12-03T02:19:58,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T02:19:59,011 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T02:19:59,017 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T02:19:59,020 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T02:19:59,021 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T02:19:59,026 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T02:19:59,041 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T02:19:59,041 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T02:19:59,042 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T02:19:59,053 INFO [RS:0;64035cbb0b46:34333 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T02:19:59,053 INFO [RS:0;64035cbb0b46:34333 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,054 INFO [RS:2;64035cbb0b46:39333 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T02:19:59,054 INFO [RS:1;64035cbb0b46:44467 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T02:19:59,054 INFO [RS:2;64035cbb0b46:39333 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,054 INFO [RS:1;64035cbb0b46:44467 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,056 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T02:19:59,056 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T02:19:59,059 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T02:19:59,063 INFO [RS:1;64035cbb0b46:44467 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T02:19:59,064 INFO [RS:2;64035cbb0b46:39333 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T02:19:59,064 INFO [RS:0;64035cbb0b46:34333 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T02:19:59,065 INFO [RS:1;64035cbb0b46:44467 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,065 INFO [RS:2;64035cbb0b46:39333 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,065 INFO [RS:0;64035cbb0b46:34333 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,066 DEBUG [RS:0;64035cbb0b46:34333 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,066 DEBUG [RS:1;64035cbb0b46:44467 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,066 DEBUG [RS:2;64035cbb0b46:39333 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,066 DEBUG [RS:0;64035cbb0b46:34333 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,066 DEBUG [RS:2;64035cbb0b46:39333 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,066 DEBUG [RS:1;64035cbb0b46:44467 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,066 DEBUG [RS:0;64035cbb0b46:34333 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,066 DEBUG [RS:2;64035cbb0b46:39333 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,066 DEBUG [RS:1;64035cbb0b46:44467 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,066 DEBUG [RS:0;64035cbb0b46:34333 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,066 DEBUG [RS:2;64035cbb0b46:39333 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,066 DEBUG [RS:1;64035cbb0b46:44467 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,066 DEBUG [RS:0;64035cbb0b46:34333 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,066 DEBUG [RS:2;64035cbb0b46:39333 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,067 DEBUG [RS:1;64035cbb0b46:44467 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,067 DEBUG [RS:2;64035cbb0b46:39333 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/64035cbb0b46:0, corePoolSize=2, maxPoolSize=2 2024-12-03T02:19:59,067 DEBUG [RS:1;64035cbb0b46:44467 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/64035cbb0b46:0, corePoolSize=2, maxPoolSize=2 2024-12-03T02:19:59,067 DEBUG [RS:0;64035cbb0b46:34333 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/64035cbb0b46:0, corePoolSize=2, maxPoolSize=2 2024-12-03T02:19:59,067 DEBUG [RS:2;64035cbb0b46:39333 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,067 DEBUG [RS:0;64035cbb0b46:34333 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,067 DEBUG [RS:2;64035cbb0b46:39333 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,067 DEBUG [RS:0;64035cbb0b46:34333 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,067 DEBUG [RS:2;64035cbb0b46:39333 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,067 DEBUG [RS:0;64035cbb0b46:34333 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,067 DEBUG [RS:2;64035cbb0b46:39333 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,067 DEBUG [RS:0;64035cbb0b46:34333 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,067 DEBUG [RS:2;64035cbb0b46:39333 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,067 DEBUG [RS:0;64035cbb0b46:34333 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,068 DEBUG [RS:2;64035cbb0b46:39333 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,068 DEBUG [RS:0;64035cbb0b46:34333 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,068 DEBUG [RS:2;64035cbb0b46:39333 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/64035cbb0b46:0, corePoolSize=3, maxPoolSize=3 2024-12-03T02:19:59,068 DEBUG [RS:1;64035cbb0b46:44467 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,068 DEBUG [RS:2;64035cbb0b46:39333 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0, corePoolSize=3, maxPoolSize=3 2024-12-03T02:19:59,068 DEBUG [RS:1;64035cbb0b46:44467 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,068 DEBUG [RS:1;64035cbb0b46:44467 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,068 DEBUG [RS:0;64035cbb0b46:34333 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/64035cbb0b46:0, corePoolSize=3, maxPoolSize=3 2024-12-03T02:19:59,068 DEBUG [RS:1;64035cbb0b46:44467 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,068 DEBUG [RS:0;64035cbb0b46:34333 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0, corePoolSize=3, maxPoolSize=3 2024-12-03T02:19:59,068 DEBUG [RS:1;64035cbb0b46:44467 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,068 DEBUG [RS:1;64035cbb0b46:44467 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:19:59,069 DEBUG [RS:1;64035cbb0b46:44467 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/64035cbb0b46:0, corePoolSize=3, maxPoolSize=3 2024-12-03T02:19:59,069 DEBUG [RS:1;64035cbb0b46:44467 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0, corePoolSize=3, maxPoolSize=3 2024-12-03T02:19:59,079 INFO [RS:0;64035cbb0b46:34333 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,080 INFO [RS:0;64035cbb0b46:34333 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,080 INFO [RS:0;64035cbb0b46:34333 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,080 INFO [RS:0;64035cbb0b46:34333 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,080 INFO [RS:0;64035cbb0b46:34333 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,080 INFO [RS:0;64035cbb0b46:34333 {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,34333,1733192397467-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T02:19:59,083 INFO [RS:2;64035cbb0b46:39333 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,083 INFO [RS:2;64035cbb0b46:39333 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,084 INFO [RS:2;64035cbb0b46:39333 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,084 INFO [RS:2;64035cbb0b46:39333 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,084 INFO [RS:2;64035cbb0b46:39333 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,084 INFO [RS:2;64035cbb0b46:39333 {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,39333,1733192397619-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T02:19:59,095 INFO [RS:1;64035cbb0b46:44467 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,095 INFO [RS:1;64035cbb0b46:44467 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,096 INFO [RS:1;64035cbb0b46:44467 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,096 INFO [RS:1;64035cbb0b46:44467 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,096 INFO [RS:1;64035cbb0b46:44467 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,096 INFO [RS:1;64035cbb0b46:44467 {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,44467,1733192397576-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T02:19:59,129 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T02:19:59,129 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T02:19:59,130 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T02:19:59,131 INFO [RS:2;64035cbb0b46:39333 {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,39333,1733192397619-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,131 INFO [RS:0;64035cbb0b46:34333 {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,34333,1733192397467-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,131 INFO [RS:1;64035cbb0b46:44467 {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,44467,1733192397576-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,132 INFO [RS:0;64035cbb0b46:34333 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,132 INFO [RS:1;64035cbb0b46:44467 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,132 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.Replication(171): 64035cbb0b46,44467,1733192397576 started 2024-12-03T02:19:59,132 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.Replication(171): 64035cbb0b46,34333,1733192397467 started 2024-12-03T02:19:59,134 INFO [RS:2;64035cbb0b46:39333 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,134 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.Replication(171): 64035cbb0b46,39333,1733192397619 started 2024-12-03T02:19:59,150 INFO [RS:0;64035cbb0b46:34333 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,151 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(1482): Serving as 64035cbb0b46,34333,1733192397467, RpcServer on 64035cbb0b46/172.17.0.3:34333, sessionid=0x1019a46de8c0001 2024-12-03T02:19:59,151 INFO [RS:2;64035cbb0b46:39333 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,151 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(1482): Serving as 64035cbb0b46,39333,1733192397619, RpcServer on 64035cbb0b46/172.17.0.3:39333, sessionid=0x1019a46de8c0003 2024-12-03T02:19:59,152 DEBUG [RS:0;64035cbb0b46:34333 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T02:19:59,152 DEBUG [RS:2;64035cbb0b46:39333 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T02:19:59,152 DEBUG [RS:0;64035cbb0b46:34333 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 64035cbb0b46,34333,1733192397467 2024-12-03T02:19:59,152 DEBUG [RS:2;64035cbb0b46:39333 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 64035cbb0b46,39333,1733192397619 2024-12-03T02:19:59,152 DEBUG [RS:0;64035cbb0b46:34333 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '64035cbb0b46,34333,1733192397467' 2024-12-03T02:19:59,152 DEBUG [RS:2;64035cbb0b46:39333 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '64035cbb0b46,39333,1733192397619' 2024-12-03T02:19:59,152 DEBUG [RS:2;64035cbb0b46:39333 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T02:19:59,152 DEBUG [RS:0;64035cbb0b46:34333 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T02:19:59,153 DEBUG [RS:2;64035cbb0b46:39333 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T02:19:59,153 DEBUG [RS:0;64035cbb0b46:34333 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T02:19:59,154 DEBUG [RS:2;64035cbb0b46:39333 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T02:19:59,154 DEBUG [RS:0;64035cbb0b46:34333 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T02:19:59,154 DEBUG [RS:0;64035cbb0b46:34333 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T02:19:59,154 DEBUG [RS:2;64035cbb0b46:39333 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T02:19:59,155 DEBUG [RS:0;64035cbb0b46:34333 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 64035cbb0b46,34333,1733192397467 2024-12-03T02:19:59,155 DEBUG [RS:2;64035cbb0b46:39333 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 64035cbb0b46,39333,1733192397619 2024-12-03T02:19:59,155 DEBUG [RS:0;64035cbb0b46:34333 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '64035cbb0b46,34333,1733192397467' 2024-12-03T02:19:59,155 DEBUG [RS:2;64035cbb0b46:39333 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '64035cbb0b46,39333,1733192397619' 2024-12-03T02:19:59,155 DEBUG [RS:2;64035cbb0b46:39333 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T02:19:59,155 DEBUG [RS:0;64035cbb0b46:34333 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T02:19:59,156 DEBUG [RS:2;64035cbb0b46:39333 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T02:19:59,156 DEBUG [RS:0;64035cbb0b46:34333 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T02:19:59,157 DEBUG [RS:2;64035cbb0b46:39333 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T02:19:59,157 DEBUG [RS:0;64035cbb0b46:34333 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T02:19:59,157 INFO [RS:2;64035cbb0b46:39333 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T02:19:59,157 INFO [RS:0;64035cbb0b46:34333 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T02:19:59,157 INFO [RS:2;64035cbb0b46:39333 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T02:19:59,157 INFO [RS:0;64035cbb0b46:34333 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T02:19:59,160 INFO [RS:1;64035cbb0b46:44467 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:19:59,160 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(1482): Serving as 64035cbb0b46,44467,1733192397576, RpcServer on 64035cbb0b46/172.17.0.3:44467, sessionid=0x1019a46de8c0002 2024-12-03T02:19:59,161 DEBUG [RS:1;64035cbb0b46:44467 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T02:19:59,161 DEBUG [RS:1;64035cbb0b46:44467 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 64035cbb0b46,44467,1733192397576 2024-12-03T02:19:59,161 DEBUG [RS:1;64035cbb0b46:44467 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '64035cbb0b46,44467,1733192397576' 2024-12-03T02:19:59,161 DEBUG [RS:1;64035cbb0b46:44467 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T02:19:59,164 DEBUG [RS:1;64035cbb0b46:44467 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T02:19:59,165 DEBUG [RS:1;64035cbb0b46:44467 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T02:19:59,165 DEBUG [RS:1;64035cbb0b46:44467 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T02:19:59,165 DEBUG [RS:1;64035cbb0b46:44467 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 64035cbb0b46,44467,1733192397576 2024-12-03T02:19:59,165 DEBUG [RS:1;64035cbb0b46:44467 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '64035cbb0b46,44467,1733192397576' 2024-12-03T02:19:59,165 DEBUG [RS:1;64035cbb0b46:44467 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T02:19:59,166 DEBUG [RS:1;64035cbb0b46:44467 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T02:19:59,166 DEBUG [RS:1;64035cbb0b46:44467 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T02:19:59,167 INFO [RS:1;64035cbb0b46:44467 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T02:19:59,167 INFO [RS:1;64035cbb0b46:44467 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T02:19:59,169 WARN [64035cbb0b46:33741 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T02:19:59,265 INFO [RS:0;64035cbb0b46:34333 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:19:59,265 INFO [RS:2;64035cbb0b46:39333 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:19:59,268 INFO [RS:1;64035cbb0b46:44467 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:19:59,270 INFO [RS:0;64035cbb0b46:34333 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=64035cbb0b46%2C34333%2C1733192397467, suffix=, logDir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467, archiveDir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/oldWALs, maxLogs=32 2024-12-03T02:19:59,272 INFO [RS:2;64035cbb0b46:39333 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=64035cbb0b46%2C39333%2C1733192397619, suffix=, logDir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,39333,1733192397619, archiveDir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/oldWALs, maxLogs=32 2024-12-03T02:19:59,273 INFO [RS:1;64035cbb0b46:44467 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=64035cbb0b46%2C44467%2C1733192397576, suffix=, logDir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,44467,1733192397576, archiveDir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/oldWALs, maxLogs=32 2024-12-03T02:19:59,294 DEBUG [RS:0;64035cbb0b46:34333 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467/64035cbb0b46%2C34333%2C1733192397467.1733192399275, exclude list is [], retry=0 2024-12-03T02:19:59,295 DEBUG [RS:1;64035cbb0b46:44467 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,44467,1733192397576/64035cbb0b46%2C44467%2C1733192397576.1733192399275, exclude list is [], retry=0 2024-12-03T02:19:59,304 DEBUG [RS:2;64035cbb0b46:39333 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,39333,1733192397619/64035cbb0b46%2C39333%2C1733192397619.1733192399275, exclude list is [], retry=0 2024-12-03T02:19:59,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:19:59,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:19:59,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:19:59,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:19:59,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:19:59,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:19:59,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:19:59,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:19:59,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:19:59,357 INFO [RS:1;64035cbb0b46:44467 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,44467,1733192397576/64035cbb0b46%2C44467%2C1733192397576.1733192399275 2024-12-03T02:19:59,362 INFO [RS:2;64035cbb0b46:39333 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,39333,1733192397619/64035cbb0b46%2C39333%2C1733192397619.1733192399275 2024-12-03T02:19:59,364 DEBUG [RS:1;64035cbb0b46:44467 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:39215:39215)] 2024-12-03T02:19:59,367 INFO [RS:0;64035cbb0b46:34333 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467/64035cbb0b46%2C34333%2C1733192397467.1733192399275 2024-12-03T02:19:59,371 DEBUG [RS:2;64035cbb0b46:39333 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:36675:36675)] 2024-12-03T02:19:59,375 DEBUG [RS:0;64035cbb0b46:34333 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:39215:39215)] 2024-12-03T02:19:59,425 DEBUG [64035cbb0b46:33741 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-03T02:19:59,438 DEBUG [64035cbb0b46:33741 {}] balancer.BalancerClusterState(204): Hosts are {64035cbb0b46=0} racks are {/default-rack=0} 2024-12-03T02:19:59,447 DEBUG [64035cbb0b46:33741 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T02:19:59,447 DEBUG [64035cbb0b46:33741 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T02:19:59,447 DEBUG [64035cbb0b46:33741 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T02:19:59,447 DEBUG [64035cbb0b46:33741 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T02:19:59,447 DEBUG [64035cbb0b46:33741 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T02:19:59,447 DEBUG [64035cbb0b46:33741 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T02:19:59,447 INFO [64035cbb0b46:33741 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T02:19:59,448 INFO [64035cbb0b46:33741 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T02:19:59,448 INFO [64035cbb0b46:33741 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T02:19:59,448 DEBUG [64035cbb0b46:33741 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T02:19:59,461 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=64035cbb0b46,44467,1733192397576 2024-12-03T02:19:59,471 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 64035cbb0b46,44467,1733192397576, state=OPENING 2024-12-03T02:19:59,489 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T02:19:59,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:59,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:59,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34333-0x1019a46de8c0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:59,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:19:59,510 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T02:19:59,510 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T02:19:59,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T02:19:59,513 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T02:19:59,515 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T02:19:59,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=64035cbb0b46,44467,1733192397576}] 2024-12-03T02:19:59,709 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T02:19:59,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52393, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T02:19:59,730 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T02:19:59,731 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:19:59,732 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T02:19:59,737 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=64035cbb0b46%2C44467%2C1733192397576.meta, suffix=.meta, logDir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,44467,1733192397576, archiveDir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/oldWALs, maxLogs=32 2024-12-03T02:19:59,757 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,44467,1733192397576/64035cbb0b46%2C44467%2C1733192397576.meta.1733192399739.meta, exclude list is [], retry=0 2024-12-03T02:19:59,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:19:59,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:19:59,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:19:59,787 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,44467,1733192397576/64035cbb0b46%2C44467%2C1733192397576.meta.1733192399739.meta 2024-12-03T02:19:59,787 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:39215:39215)] 2024-12-03T02:19:59,787 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:19:59,790 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T02:19:59,792 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T02:19:59,798 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T02:19:59,804 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T02:19:59,804 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:19:59,805 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T02:19:59,805 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T02:19:59,808 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T02:19:59,810 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T02:19:59,810 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:19:59,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:19:59,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T02:19:59,814 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T02:19:59,814 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:19:59,815 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:19:59,815 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T02:19:59,817 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T02:19:59,817 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:19:59,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:19:59,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T02:19:59,820 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T02:19:59,820 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:19:59,821 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:19:59,822 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T02:19:59,823 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740 2024-12-03T02:19:59,826 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740 2024-12-03T02:19:59,829 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T02:19:59,829 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T02:19:59,830 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T02:19:59,832 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T02:19:59,834 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70806526, jitterRate=0.05509945750236511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T02:19:59,834 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T02:19:59,835 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733192399805Writing region info on filesystem at 1733192399806 (+1 ms)Initializing all the Stores at 1733192399808 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733192399808Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733192399808Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192399808Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733192399808Cleaning up temporary data from old regions at 1733192399829 (+21 ms)Running coprocessor post-open hooks at 1733192399834 (+5 ms)Region opened successfully at 1733192399835 (+1 ms) 2024-12-03T02:19:59,847 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733192399695 2024-12-03T02:19:59,858 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T02:19:59,859 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T02:19:59,861 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=64035cbb0b46,44467,1733192397576 2024-12-03T02:19:59,863 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 64035cbb0b46,44467,1733192397576, state=OPEN 2024-12-03T02:19:59,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T02:19:59,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34333-0x1019a46de8c0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T02:19:59,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T02:19:59,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T02:19:59,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T02:19:59,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T02:19:59,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T02:19:59,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T02:19:59,915 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=64035cbb0b46,44467,1733192397576 2024-12-03T02:19:59,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T02:19:59,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=64035cbb0b46,44467,1733192397576 in 400 msec 2024-12-03T02:19:59,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T02:19:59,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 947 msec 2024-12-03T02:19:59,950 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T02:19:59,951 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T02:19:59,977 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:19:59,979 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,44467,1733192397576, seqNum=-1] 2024-12-03T02:19:59,997 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:19:59,999 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54843, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:20:00,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3640 sec 2024-12-03T02:20:00,026 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733192400025, completionTime=-1 2024-12-03T02:20:00,028 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-03T02:20:00,028 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T02:20:00,053 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-03T02:20:00,053 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733192460053 2024-12-03T02:20:00,053 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733192520053 2024-12-03T02:20:00,053 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-12-03T02:20:00,055 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-03T02:20:00,062 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,33741,1733192396696-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:20:00,062 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,33741,1733192396696-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:20:00,062 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,33741,1733192396696-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:20:00,064 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-64035cbb0b46:33741, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:20:00,064 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T02:20:00,064 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T02:20:00,071 DEBUG [master/64035cbb0b46:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T02:20:00,091 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.367sec 2024-12-03T02:20:00,093 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T02:20:00,094 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T02:20:00,095 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T02:20:00,095 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T02:20:00,095 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T02:20:00,096 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,33741,1733192396696-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T02:20:00,096 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,33741,1733192396696-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T02:20:00,100 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T02:20:00,101 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T02:20:00,102 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,33741,1733192396696-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:20:00,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70faddef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:20:00,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,33741,-1 for getting cluster id 2024-12-03T02:20:00,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:20:00,173 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1be2a799-8e6e-40d2-897e-f78b191efd08' 2024-12-03T02:20:00,175 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:20:00,175 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1be2a799-8e6e-40d2-897e-f78b191efd08" 2024-12-03T02:20:00,176 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60810180, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:20:00,176 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,33741,-1] 2024-12-03T02:20:00,179 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:20:00,182 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:20:00,182 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:20:00,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35a92b38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:20:00,186 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:20:00,192 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,44467,1733192397576, seqNum=-1] 2024-12-03T02:20:00,193 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:20:00,195 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:20:00,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=64035cbb0b46,33741,1733192396696 2024-12-03T02:20:00,212 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:44855/hbase 2024-12-03T02:20:00,228 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=368, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=416, ProcessCount=11, AvailableMemoryMB=463 2024-12-03T02:20:00,246 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:00,249 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:00,250 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:20:00,255 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-63504299, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/hregion-63504299, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:00,269 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-63504299/hregion-63504299.1733192400256, exclude list is [], retry=0 2024-12-03T02:20:00,273 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:00,273 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:00,274 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:00,276 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-63504299/hregion-63504299.1733192400256 2024-12-03T02:20:00,280 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:36675:36675)] 2024-12-03T02:20:00,280 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ab67ba1869b6175027d56f45b1fceacb, NAME => 'testReplayEditsWrittenIntoWAL,,1733192400247.ab67ba1869b6175027d56f45b1fceacb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44855/hbase 2024-12-03T02:20:00,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741838_1014 (size=64) 2024-12-03T02:20:00,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741838_1014 (size=64) 2024-12-03T02:20:00,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741838_1014 (size=64) 2024-12-03T02:20:00,295 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733192400247.ab67ba1869b6175027d56f45b1fceacb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:00,296 INFO [StoreOpener-ab67ba1869b6175027d56f45b1fceacb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ab67ba1869b6175027d56f45b1fceacb 2024-12-03T02:20:00,298 INFO [StoreOpener-ab67ba1869b6175027d56f45b1fceacb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab67ba1869b6175027d56f45b1fceacb columnFamilyName a 2024-12-03T02:20:00,298 DEBUG [StoreOpener-ab67ba1869b6175027d56f45b1fceacb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:00,299 INFO [StoreOpener-ab67ba1869b6175027d56f45b1fceacb-1 {}] regionserver.HStore(327): Store=ab67ba1869b6175027d56f45b1fceacb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:00,300 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ab67ba1869b6175027d56f45b1fceacb 2024-12-03T02:20:00,301 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb 2024-12-03T02:20:00,301 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb 2024-12-03T02:20:00,302 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ab67ba1869b6175027d56f45b1fceacb 2024-12-03T02:20:00,302 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ab67ba1869b6175027d56f45b1fceacb 2024-12-03T02:20:00,305 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ab67ba1869b6175027d56f45b1fceacb 2024-12-03T02:20:00,309 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:20:00,310 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ab67ba1869b6175027d56f45b1fceacb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71308145, jitterRate=0.06257416307926178}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T02:20:00,311 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ab67ba1869b6175027d56f45b1fceacb: Writing region info on filesystem at 1733192400295Initializing all the Stores at 1733192400296 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192400296Cleaning up temporary data from old regions at 1733192400302 (+6 ms)Region opened successfully at 1733192400311 (+9 ms) 2024-12-03T02:20:00,311 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ab67ba1869b6175027d56f45b1fceacb, disabling compactions & flushes 2024-12-03T02:20:00,311 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733192400247.ab67ba1869b6175027d56f45b1fceacb. 2024-12-03T02:20:00,311 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733192400247.ab67ba1869b6175027d56f45b1fceacb. 2024-12-03T02:20:00,311 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733192400247.ab67ba1869b6175027d56f45b1fceacb. after waiting 0 ms 2024-12-03T02:20:00,312 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733192400247.ab67ba1869b6175027d56f45b1fceacb. 2024-12-03T02:20:00,312 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733192400247.ab67ba1869b6175027d56f45b1fceacb. 2024-12-03T02:20:00,312 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ab67ba1869b6175027d56f45b1fceacb: Waiting for close lock at 1733192400311Disabling compacts and flushes for region at 1733192400311Disabling writes for close at 1733192400312 (+1 ms)Writing region close event to WAL at 1733192400312Closed at 1733192400312 2024-12-03T02:20:00,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741837_1013 (size=93) 2024-12-03T02:20:00,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741837_1013 (size=93) 2024-12-03T02:20:00,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741837_1013 (size=93) 2024-12-03T02:20:00,323 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T02:20:00,323 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-63504299:(num 1733192400256) 2024-12-03T02:20:00,325 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T02:20:00,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741839_1015 (size=369) 2024-12-03T02:20:00,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741839_1015 (size=369) 2024-12-03T02:20:00,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741839_1015 (size=369) 2024-12-03T02:20:00,344 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T02:20:00,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741840_1016 (size=231) 2024-12-03T02:20:00,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741840_1016 (size=231) 2024-12-03T02:20:00,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741840_1016 (size=231) 2024-12-03T02:20:00,382 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-1, size=369 (369bytes) 2024-12-03T02:20:00,383 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-03T02:20:00,383 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-03T02:20:00,383 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-1 2024-12-03T02:20:00,389 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-1 after 4ms 2024-12-03T02:20:00,395 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-1 took 14ms 2024-12-03T02:20:00,399 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-1 so closing down 2024-12-03T02:20:00,402 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-03T02:20:00,403 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T02:20:00,404 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/0000000000000000001-wal-1.temp 2024-12-03T02:20:00,406 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T02:20:00,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741841_1017 (size=369) 2024-12-03T02:20:00,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741841_1017 (size=369) 2024-12-03T02:20:00,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741841_1017 (size=369) 2024-12-03T02:20:00,421 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T02:20:00,425 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/0000000000000000002 2024-12-03T02:20:00,429 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 29 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-12-03T02:20:00,429 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-1, journal: Splitting hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-1, size=369 (369bytes) at 1733192400383Finishing writing output for hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-1 so closing down at 1733192400399 (+16 ms)Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/0000000000000000001-wal-1.temp at 1733192400404 (+5 ms)3 split writer threads finished at 1733192400406 (+2 ms)Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733192400421 (+15 ms)Rename recovered edits hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/0000000000000000002 at 1733192400425 (+4 ms)Processed 2 edits across 1 Regions in 29 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1733192400429 (+4 ms) 2024-12-03T02:20:00,442 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-2, size=231 (231bytes) 2024-12-03T02:20:00,443 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-2 2024-12-03T02:20:00,443 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-2 after 0ms 2024-12-03T02:20:00,447 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-2 took 5ms 2024-12-03T02:20:00,450 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-2 so closing down 2024-12-03T02:20:00,450 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T02:20:00,452 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-03T02:20:00,454 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/0000000000000000002-wal-2.temp 2024-12-03T02:20:00,454 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T02:20:00,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741842_1018 (size=231) 2024-12-03T02:20:00,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741842_1018 (size=231) 2024-12-03T02:20:00,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741842_1018 (size=231) 2024-12-03T02:20:00,465 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T02:20:00,472 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-12-03T02:20:00,474 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-12-03T02:20:00,474 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-2, journal: Splitting hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-2, size=231 (231bytes) at 1733192400443Finishing writing output for hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-2 so closing down at 1733192400450 (+7 ms)Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/0000000000000000002-wal-2.temp at 1733192400454 (+4 ms)3 split writer threads finished at 1733192400454Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733192400465 (+11 ms)Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1733192400474 (+9 ms) 2024-12-03T02:20:00,474 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:00,476 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:00,488 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal.1733192400477, exclude list is [], retry=0 2024-12-03T02:20:00,492 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:00,492 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:00,493 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:00,494 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal.1733192400477 2024-12-03T02:20:00,495 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:36675:36675)] 2024-12-03T02:20:00,495 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ab67ba1869b6175027d56f45b1fceacb, NAME => 'testReplayEditsWrittenIntoWAL,,1733192400247.ab67ba1869b6175027d56f45b1fceacb.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:00,495 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733192400247.ab67ba1869b6175027d56f45b1fceacb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:00,495 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ab67ba1869b6175027d56f45b1fceacb 2024-12-03T02:20:00,495 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ab67ba1869b6175027d56f45b1fceacb 2024-12-03T02:20:00,498 INFO [StoreOpener-ab67ba1869b6175027d56f45b1fceacb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ab67ba1869b6175027d56f45b1fceacb 2024-12-03T02:20:00,499 INFO [StoreOpener-ab67ba1869b6175027d56f45b1fceacb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab67ba1869b6175027d56f45b1fceacb columnFamilyName a 2024-12-03T02:20:00,499 DEBUG [StoreOpener-ab67ba1869b6175027d56f45b1fceacb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:00,501 INFO [StoreOpener-ab67ba1869b6175027d56f45b1fceacb-1 {}] regionserver.HStore(327): Store=ab67ba1869b6175027d56f45b1fceacb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:00,501 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ab67ba1869b6175027d56f45b1fceacb 2024-12-03T02:20:00,502 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb 2024-12-03T02:20:00,505 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb 2024-12-03T02:20:00,506 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/0000000000000000002 2024-12-03T02:20:00,515 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/0000000000000000002 2024-12-03T02:20:00,518 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ab67ba1869b6175027d56f45b1fceacb 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-03T02:20:00,567 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/.tmp/a/370291470f9c4ba99449c41abc3b8439 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733192400323/Put/seqid=0 2024-12-03T02:20:00,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741844_1020 (size=5170) 2024-12-03T02:20:00,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741844_1020 (size=5170) 2024-12-03T02:20:00,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741844_1020 (size=5170) 2024-12-03T02:20:00,580 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/.tmp/a/370291470f9c4ba99449c41abc3b8439 2024-12-03T02:20:00,621 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/.tmp/a/370291470f9c4ba99449c41abc3b8439 as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/a/370291470f9c4ba99449c41abc3b8439 2024-12-03T02:20:00,629 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/a/370291470f9c4ba99449c41abc3b8439, entries=2, sequenceid=2, filesize=5.0 K 2024-12-03T02:20:00,635 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for ab67ba1869b6175027d56f45b1fceacb in 115ms, sequenceid=2, compaction requested=false; wal=null 2024-12-03T02:20:00,638 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/0000000000000000002 2024-12-03T02:20:00,638 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ab67ba1869b6175027d56f45b1fceacb 2024-12-03T02:20:00,638 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ab67ba1869b6175027d56f45b1fceacb 2024-12-03T02:20:00,643 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ab67ba1869b6175027d56f45b1fceacb 2024-12-03T02:20:00,646 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/ab67ba1869b6175027d56f45b1fceacb/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-03T02:20:00,647 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ab67ba1869b6175027d56f45b1fceacb; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75414797, jitterRate=0.12376804649829865}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T02:20:00,649 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ab67ba1869b6175027d56f45b1fceacb: Writing region info on filesystem at 1733192400496Initializing all the Stores at 1733192400497 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192400497Obtaining lock to block concurrent updates at 1733192400518 (+21 ms)Preparing flush snapshotting stores in ab67ba1869b6175027d56f45b1fceacb at 1733192400518Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733192400247.ab67ba1869b6175027d56f45b1fceacb., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733192400521 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733192400247.ab67ba1869b6175027d56f45b1fceacb. at 1733192400521Flushing ab67ba1869b6175027d56f45b1fceacb/a: creating writer at 1733192400522 (+1 ms)Flushing ab67ba1869b6175027d56f45b1fceacb/a: appending metadata at 1733192400557 (+35 ms)Flushing ab67ba1869b6175027d56f45b1fceacb/a: closing flushed file at 1733192400560 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f5a647e: reopening flushed file at 1733192400619 (+59 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for ab67ba1869b6175027d56f45b1fceacb in 115ms, sequenceid=2, compaction requested=false; wal=null at 1733192400636 (+17 ms)Cleaning up temporary data from old regions at 1733192400639 (+3 ms)Region opened successfully at 1733192400648 (+9 ms) 2024-12-03T02:20:00,678 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=380 (was 368) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:54424 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-537057715-172.17.0.3-1733192391930:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-537057715-172.17.0.3-1733192391930:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:44855/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-537057715-172.17.0.3-1733192391930:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:44160 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:54276 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:49090 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:44176 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=415 (was 416), ProcessCount=11 (was 11), AvailableMemoryMB=458 (was 463) 2024-12-03T02:20:00,689 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=380, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=415, ProcessCount=11, AvailableMemoryMB=458 2024-12-03T02:20:00,708 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:00,711 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:00,712 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:20:00,716 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-35384493, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/hregion-35384493, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:00,729 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-35384493/hregion-35384493.1733192400716, exclude list is [], retry=0 2024-12-03T02:20:00,734 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:00,734 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:00,735 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:00,739 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-35384493/hregion-35384493.1733192400716 2024-12-03T02:20:00,739 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:36675:36675)] 2024-12-03T02:20:00,739 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3d9fe05bb7ca9058b37c3ed461bd659f, NAME => 'testReplayEditsWrittenIntoWAL,,1733192400709.3d9fe05bb7ca9058b37c3ed461bd659f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44855/hbase 2024-12-03T02:20:00,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741846_1022 (size=64) 2024-12-03T02:20:00,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741846_1022 (size=64) 2024-12-03T02:20:00,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741846_1022 (size=64) 2024-12-03T02:20:00,752 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733192400709.3d9fe05bb7ca9058b37c3ed461bd659f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:00,754 INFO [StoreOpener-3d9fe05bb7ca9058b37c3ed461bd659f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3d9fe05bb7ca9058b37c3ed461bd659f 2024-12-03T02:20:00,756 INFO [StoreOpener-3d9fe05bb7ca9058b37c3ed461bd659f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d9fe05bb7ca9058b37c3ed461bd659f columnFamilyName a 2024-12-03T02:20:00,757 DEBUG [StoreOpener-3d9fe05bb7ca9058b37c3ed461bd659f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:00,757 INFO [StoreOpener-3d9fe05bb7ca9058b37c3ed461bd659f-1 {}] regionserver.HStore(327): Store=3d9fe05bb7ca9058b37c3ed461bd659f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:00,757 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3d9fe05bb7ca9058b37c3ed461bd659f 2024-12-03T02:20:00,758 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f 2024-12-03T02:20:00,759 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f 2024-12-03T02:20:00,759 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3d9fe05bb7ca9058b37c3ed461bd659f 2024-12-03T02:20:00,759 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3d9fe05bb7ca9058b37c3ed461bd659f 2024-12-03T02:20:00,762 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3d9fe05bb7ca9058b37c3ed461bd659f 2024-12-03T02:20:00,765 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:20:00,766 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3d9fe05bb7ca9058b37c3ed461bd659f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75013871, jitterRate=0.11779378354549408}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T02:20:00,766 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3d9fe05bb7ca9058b37c3ed461bd659f: Writing region info on filesystem at 1733192400752Initializing all the Stores at 1733192400754 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192400754Cleaning up temporary data from old regions at 1733192400759 (+5 ms)Region opened successfully at 1733192400766 (+7 ms) 2024-12-03T02:20:00,766 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3d9fe05bb7ca9058b37c3ed461bd659f, disabling compactions & flushes 2024-12-03T02:20:00,766 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733192400709.3d9fe05bb7ca9058b37c3ed461bd659f. 2024-12-03T02:20:00,766 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733192400709.3d9fe05bb7ca9058b37c3ed461bd659f. 2024-12-03T02:20:00,766 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733192400709.3d9fe05bb7ca9058b37c3ed461bd659f. after waiting 0 ms 2024-12-03T02:20:00,766 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733192400709.3d9fe05bb7ca9058b37c3ed461bd659f. 2024-12-03T02:20:00,766 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733192400709.3d9fe05bb7ca9058b37c3ed461bd659f. 2024-12-03T02:20:00,767 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3d9fe05bb7ca9058b37c3ed461bd659f: Waiting for close lock at 1733192400766Disabling compacts and flushes for region at 1733192400766Disabling writes for close at 1733192400766Writing region close event to WAL at 1733192400766Closed at 1733192400766 2024-12-03T02:20:00,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741845_1021 (size=93) 2024-12-03T02:20:00,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741845_1021 (size=93) 2024-12-03T02:20:00,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741845_1021 (size=93) 2024-12-03T02:20:00,773 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T02:20:00,773 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-35384493:(num 1733192400716) 2024-12-03T02:20:00,774 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T02:20:00,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741847_1023 (size=369) 2024-12-03T02:20:00,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741847_1023 (size=369) 2024-12-03T02:20:00,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741847_1023 (size=369) 2024-12-03T02:20:00,795 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T02:20:00,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741848_1024 (size=231) 2024-12-03T02:20:00,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741848_1024 (size=231) 2024-12-03T02:20:00,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741848_1024 (size=231) 2024-12-03T02:20:00,821 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-2, size=231 (231bytes) 2024-12-03T02:20:00,821 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-2 2024-12-03T02:20:00,822 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-2 after 1ms 2024-12-03T02:20:00,825 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-2 took 4ms 2024-12-03T02:20:00,827 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-2 so closing down 2024-12-03T02:20:00,827 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T02:20:00,830 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-03T02:20:00,832 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000002-wal-2.temp 2024-12-03T02:20:00,832 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T02:20:00,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741849_1025 (size=231) 2024-12-03T02:20:00,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741849_1025 (size=231) 2024-12-03T02:20:00,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741849_1025 (size=231) 2024-12-03T02:20:00,843 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T02:20:00,845 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000002 2024-12-03T02:20:00,845 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-12-03T02:20:00,845 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-2, journal: Splitting hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-2, size=231 (231bytes) at 1733192400821Finishing writing output for hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-2 so closing down at 1733192400827 (+6 ms)Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000002-wal-2.temp at 1733192400832 (+5 ms)3 split writer threads finished at 1733192400832Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733192400843 (+11 ms)Rename recovered edits hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000002 at 1733192400845 (+2 ms)Processed 1 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1733192400845 2024-12-03T02:20:00,858 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-1, size=369 (369bytes) 2024-12-03T02:20:00,858 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-1 2024-12-03T02:20:00,859 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-1 after 1ms 2024-12-03T02:20:00,863 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-1 took 5ms 2024-12-03T02:20:00,866 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-1 so closing down 2024-12-03T02:20:00,866 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T02:20:00,869 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-03T02:20:00,870 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000001-wal-1.temp 2024-12-03T02:20:00,871 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T02:20:00,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741850_1026 (size=369) 2024-12-03T02:20:00,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741850_1026 (size=369) 2024-12-03T02:20:00,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741850_1026 (size=369) 2024-12-03T02:20:00,881 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T02:20:00,887 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000002, length=231 2024-12-03T02:20:00,890 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000002 2024-12-03T02:20:00,890 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 27 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-12-03T02:20:00,890 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-1, journal: Splitting hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-1, size=369 (369bytes) at 1733192400858Finishing writing output for hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-1 so closing down at 1733192400866 (+8 ms)Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000001-wal-1.temp at 1733192400870 (+4 ms)3 split writer threads finished at 1733192400871 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733192400881 (+10 ms)Rename recovered edits hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000002 at 1733192400890 (+9 ms)Processed 2 edits across 1 Regions in 27 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1733192400890 2024-12-03T02:20:00,890 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:00,894 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:00,908 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal.1733192400895, exclude list is [], retry=0 2024-12-03T02:20:00,912 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:00,912 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:00,913 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:00,915 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal.1733192400895 2024-12-03T02:20:00,915 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:36675:36675)] 2024-12-03T02:20:00,915 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3d9fe05bb7ca9058b37c3ed461bd659f, NAME => 'testReplayEditsWrittenIntoWAL,,1733192400709.3d9fe05bb7ca9058b37c3ed461bd659f.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:00,915 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733192400709.3d9fe05bb7ca9058b37c3ed461bd659f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:00,915 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3d9fe05bb7ca9058b37c3ed461bd659f 2024-12-03T02:20:00,915 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3d9fe05bb7ca9058b37c3ed461bd659f 2024-12-03T02:20:00,917 INFO [StoreOpener-3d9fe05bb7ca9058b37c3ed461bd659f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3d9fe05bb7ca9058b37c3ed461bd659f 2024-12-03T02:20:00,920 INFO [StoreOpener-3d9fe05bb7ca9058b37c3ed461bd659f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d9fe05bb7ca9058b37c3ed461bd659f columnFamilyName a 2024-12-03T02:20:00,920 DEBUG [StoreOpener-3d9fe05bb7ca9058b37c3ed461bd659f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:00,921 INFO [StoreOpener-3d9fe05bb7ca9058b37c3ed461bd659f-1 {}] regionserver.HStore(327): Store=3d9fe05bb7ca9058b37c3ed461bd659f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:00,921 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3d9fe05bb7ca9058b37c3ed461bd659f 2024-12-03T02:20:00,922 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f 2024-12-03T02:20:00,924 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f 2024-12-03T02:20:00,925 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000002 2024-12-03T02:20:00,929 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000002 2024-12-03T02:20:00,930 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3d9fe05bb7ca9058b37c3ed461bd659f 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-03T02:20:00,946 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/.tmp/a/39ef0c5181ab4af488a3264f8bef349b is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733192400773/Put/seqid=0 2024-12-03T02:20:00,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741852_1028 (size=5170) 2024-12-03T02:20:00,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741852_1028 (size=5170) 2024-12-03T02:20:00,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741852_1028 (size=5170) 2024-12-03T02:20:00,959 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/.tmp/a/39ef0c5181ab4af488a3264f8bef349b 2024-12-03T02:20:00,971 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/.tmp/a/39ef0c5181ab4af488a3264f8bef349b as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/a/39ef0c5181ab4af488a3264f8bef349b 2024-12-03T02:20:00,981 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/a/39ef0c5181ab4af488a3264f8bef349b, entries=2, sequenceid=2, filesize=5.0 K 2024-12-03T02:20:00,981 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 3d9fe05bb7ca9058b37c3ed461bd659f in 52ms, sequenceid=2, compaction requested=false; wal=null 2024-12-03T02:20:00,982 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/0000000000000000002 2024-12-03T02:20:00,983 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3d9fe05bb7ca9058b37c3ed461bd659f 2024-12-03T02:20:00,983 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3d9fe05bb7ca9058b37c3ed461bd659f 2024-12-03T02:20:00,986 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3d9fe05bb7ca9058b37c3ed461bd659f 2024-12-03T02:20:00,990 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/3d9fe05bb7ca9058b37c3ed461bd659f/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-03T02:20:00,991 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3d9fe05bb7ca9058b37c3ed461bd659f; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70869750, jitterRate=0.056041568517684937}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T02:20:00,991 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3d9fe05bb7ca9058b37c3ed461bd659f: Writing region info on filesystem at 1733192400916Initializing all the Stores at 1733192400917 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192400917Obtaining lock to block concurrent updates at 1733192400930 (+13 ms)Preparing flush snapshotting stores in 3d9fe05bb7ca9058b37c3ed461bd659f at 1733192400930Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733192400709.3d9fe05bb7ca9058b37c3ed461bd659f., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733192400930Flushing stores of testReplayEditsWrittenIntoWAL,,1733192400709.3d9fe05bb7ca9058b37c3ed461bd659f. at 1733192400930Flushing 3d9fe05bb7ca9058b37c3ed461bd659f/a: creating writer at 1733192400930Flushing 3d9fe05bb7ca9058b37c3ed461bd659f/a: appending metadata at 1733192400945 (+15 ms)Flushing 3d9fe05bb7ca9058b37c3ed461bd659f/a: closing flushed file at 1733192400945Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a16ae71: reopening flushed file at 1733192400969 (+24 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 3d9fe05bb7ca9058b37c3ed461bd659f in 52ms, sequenceid=2, compaction requested=false; wal=null at 1733192400981 (+12 ms)Cleaning up temporary data from old regions at 1733192400983 (+2 ms)Region opened successfully at 1733192400991 (+8 ms) 2024-12-03T02:20:01,010 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=390 (was 380) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:49160 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:48940 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-537057715-172.17.0.3-1733192391930:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-537057715-172.17.0.3-1733192391930:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:44160 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:54276 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:44260 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:54474 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-537057715-172.17.0.3-1733192391930:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=415 (was 415), ProcessCount=11 (was 11), AvailableMemoryMB=448 (was 458) 2024-12-03T02:20:01,020 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=390, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=415, ProcessCount=11, AvailableMemoryMB=447 2024-12-03T02:20:01,035 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:01,037 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:01,038 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:20:01,041 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-81957961, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/hregion-81957961, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:01,053 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-81957961/hregion-81957961.1733192401041, exclude list is [], retry=0 2024-12-03T02:20:01,057 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:01,057 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:01,057 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:01,059 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-81957961/hregion-81957961.1733192401041 2024-12-03T02:20:01,060 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:39215:39215)] 2024-12-03T02:20:01,060 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a3af9ffcc40e129486f79112d9915f21, NAME => 'testReplayEditsWrittenIntoWAL,,1733192401035.a3af9ffcc40e129486f79112d9915f21.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44855/hbase 2024-12-03T02:20:01,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741854_1030 (size=64) 2024-12-03T02:20:01,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741854_1030 (size=64) 2024-12-03T02:20:01,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741854_1030 (size=64) 2024-12-03T02:20:01,074 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733192401035.a3af9ffcc40e129486f79112d9915f21.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:01,076 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:01,079 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3af9ffcc40e129486f79112d9915f21 columnFamilyName a 2024-12-03T02:20:01,079 DEBUG [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:01,080 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] regionserver.HStore(327): Store=a3af9ffcc40e129486f79112d9915f21/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:01,080 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:01,082 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3af9ffcc40e129486f79112d9915f21 columnFamilyName b 2024-12-03T02:20:01,082 DEBUG [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:01,083 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] regionserver.HStore(327): Store=a3af9ffcc40e129486f79112d9915f21/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:01,083 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:01,085 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3af9ffcc40e129486f79112d9915f21 columnFamilyName c 2024-12-03T02:20:01,085 DEBUG [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:01,086 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] regionserver.HStore(327): Store=a3af9ffcc40e129486f79112d9915f21/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:01,086 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:01,087 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:01,088 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:01,089 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:01,089 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:01,090 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:01,092 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:01,095 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:20:01,095 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a3af9ffcc40e129486f79112d9915f21; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61472756, jitterRate=-0.08398455381393433}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:01,096 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a3af9ffcc40e129486f79112d9915f21: Writing region info on filesystem at 1733192401074Initializing all the Stores at 1733192401076 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192401076Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192401076Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192401076Cleaning up temporary data from old regions at 1733192401089 (+13 ms)Region opened successfully at 1733192401095 (+6 ms) 2024-12-03T02:20:01,096 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a3af9ffcc40e129486f79112d9915f21, disabling compactions & flushes 2024-12-03T02:20:01,096 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733192401035.a3af9ffcc40e129486f79112d9915f21. 2024-12-03T02:20:01,096 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733192401035.a3af9ffcc40e129486f79112d9915f21. 2024-12-03T02:20:01,096 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733192401035.a3af9ffcc40e129486f79112d9915f21. after waiting 0 ms 2024-12-03T02:20:01,096 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733192401035.a3af9ffcc40e129486f79112d9915f21. 2024-12-03T02:20:01,096 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733192401035.a3af9ffcc40e129486f79112d9915f21. 2024-12-03T02:20:01,096 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a3af9ffcc40e129486f79112d9915f21: Waiting for close lock at 1733192401096Disabling compacts and flushes for region at 1733192401096Disabling writes for close at 1733192401096Writing region close event to WAL at 1733192401096Closed at 1733192401096 2024-12-03T02:20:01,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741853_1029 (size=93) 2024-12-03T02:20:01,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741853_1029 (size=93) 2024-12-03T02:20:01,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741853_1029 (size=93) 2024-12-03T02:20:01,103 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T02:20:01,103 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-81957961:(num 1733192401041) 2024-12-03T02:20:01,103 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:01,106 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:01,119 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107, exclude list is [], retry=0 2024-12-03T02:20:01,122 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:01,122 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:01,123 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:01,125 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107 2024-12-03T02:20:01,126 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:43847:43847)] 2024-12-03T02:20:01,347 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107, size=0 (0bytes) 2024-12-03T02:20:01,347 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107 might be still open, length is 0 2024-12-03T02:20:01,347 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107 2024-12-03T02:20:01,348 WARN [IPC Server handler 1 on default port 44855 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-03T02:20:01,349 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107 after 2ms 2024-12-03T02:20:01,784 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:49178 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:35125:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49178 dst: /127.0.0.1:35125 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35125 remote=/127.0.0.1:49178]. Total timeout mills is 60000, 59522 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:01,785 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:54522 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43037:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54522 dst: /127.0.0.1:43037 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:01,785 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:44290 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36885:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44290 dst: /127.0.0.1:36885 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:01,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741855_1032 (size=470675) 2024-12-03T02:20:01,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741855_1032 (size=470675) 2024-12-03T02:20:01,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741855_1032 (size=470675) 2024-12-03T02:20:05,320 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T02:20:05,350 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107 after 4003ms 2024-12-03T02:20:05,356 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107 took 4009ms 2024-12-03T02:20:05,361 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733192401107.temp 2024-12-03T02:20:05,369 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/recovered.edits/0000000000000000001-wal.1733192401107.temp 2024-12-03T02:20:05,372 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T02:20:05,441 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107; continuing. 2024-12-03T02:20:05,441 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107 so closing down 2024-12-03T02:20:05,441 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T02:20:05,444 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T02:20:05,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741856_1033 (size=470683) 2024-12-03T02:20:05,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741856_1033 (size=470683) 2024-12-03T02:20:05,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741856_1033 (size=470683) 2024-12-03T02:20:05,460 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/recovered.edits/0000000000000000001-wal.1733192401107.temp (wrote 3002 edits, skipped 0 edits in 26 ms) 2024-12-03T02:20:05,463 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/recovered.edits/0000000000000000001-wal.1733192401107.temp to hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/recovered.edits/0000000000000003002 2024-12-03T02:20:05,463 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 107 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T02:20:05,463 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107, journal: Splitting hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107, size=0 (0bytes) at 1733192401347Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/recovered.edits/0000000000000000001-wal.1733192401107.temp at 1733192405369 (+4022 ms)Split 1024 edits, skipped 0 edits. at 1733192405407 (+38 ms)Split 2048 edits, skipped 0 edits. at 1733192405425 (+18 ms)Finishing writing output for hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107 so closing down at 1733192405441 (+16 ms)3 split writer threads finished at 1733192405444 (+3 ms)Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/recovered.edits/0000000000000000001-wal.1733192401107.temp (wrote 3002 edits, skipped 0 edits in 26 ms) at 1733192405460 (+16 ms)Rename recovered edits hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/recovered.edits/0000000000000000001-wal.1733192401107.temp to hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/recovered.edits/0000000000000003002 at 1733192405463 (+3 ms)Processed 3002 edits across 1 Regions in 107 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107, size=0, length=0, corrupted=false, cancelled=false at 1733192405463 2024-12-03T02:20:05,467 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107 to hdfs://localhost:44855/hbase/oldWALs/wal.1733192401107 2024-12-03T02:20:05,469 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/recovered.edits/0000000000000003002 2024-12-03T02:20:05,469 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:05,472 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:05,486 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192405473, exclude list is [], retry=0 2024-12-03T02:20:05,491 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:05,491 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:05,491 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:05,500 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192405473 2024-12-03T02:20:05,500 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:39215:39215)] 2024-12-03T02:20:05,500 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733192401035.a3af9ffcc40e129486f79112d9915f21.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:05,502 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:05,504 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3af9ffcc40e129486f79112d9915f21 columnFamilyName a 2024-12-03T02:20:05,504 DEBUG [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:05,505 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] regionserver.HStore(327): Store=a3af9ffcc40e129486f79112d9915f21/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:05,505 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:05,506 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3af9ffcc40e129486f79112d9915f21 columnFamilyName b 2024-12-03T02:20:05,507 DEBUG [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:05,507 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] regionserver.HStore(327): Store=a3af9ffcc40e129486f79112d9915f21/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:05,508 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:05,509 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3af9ffcc40e129486f79112d9915f21 columnFamilyName c 2024-12-03T02:20:05,509 DEBUG [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:05,510 INFO [StoreOpener-a3af9ffcc40e129486f79112d9915f21-1 {}] regionserver.HStore(327): Store=a3af9ffcc40e129486f79112d9915f21/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:05,510 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:05,511 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:05,515 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:05,516 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/recovered.edits/0000000000000003002 2024-12-03T02:20:05,565 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T02:20:05,958 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a3af9ffcc40e129486f79112d9915f21 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T02:20:06,006 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/a/eb619c6c603442579b94deafd9d1d4c2 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733192401132/Put/seqid=0 2024-12-03T02:20:06,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741858_1035 (size=50463) 2024-12-03T02:20:06,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741858_1035 (size=50463) 2024-12-03T02:20:06,017 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/a/eb619c6c603442579b94deafd9d1d4c2 2024-12-03T02:20:06,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741858_1035 (size=50463) 2024-12-03T02:20:06,027 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/a/eb619c6c603442579b94deafd9d1d4c2 as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/a/eb619c6c603442579b94deafd9d1d4c2 2024-12-03T02:20:06,041 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/a/eb619c6c603442579b94deafd9d1d4c2, entries=754, sequenceid=754, filesize=49.3 K 2024-12-03T02:20:06,042 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for a3af9ffcc40e129486f79112d9915f21 in 84ms, sequenceid=754, compaction requested=false; wal=null 2024-12-03T02:20:06,054 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T02:20:06,054 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a3af9ffcc40e129486f79112d9915f21 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T02:20:06,088 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/a/31a6f1ea2e6d414ab240c089637559ec is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733192401177/Put/seqid=0 2024-12-03T02:20:06,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741859_1036 (size=20072) 2024-12-03T02:20:06,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741859_1036 (size=20072) 2024-12-03T02:20:06,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741859_1036 (size=20072) 2024-12-03T02:20:06,110 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/a/31a6f1ea2e6d414ab240c089637559ec 2024-12-03T02:20:06,133 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/b/0d559581a66c493eb8b82f736aa82dfa is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733192401202/Put/seqid=0 2024-12-03T02:20:06,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741860_1037 (size=35835) 2024-12-03T02:20:06,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741860_1037 (size=35835) 2024-12-03T02:20:06,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741860_1037 (size=35835) 2024-12-03T02:20:06,141 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/b/0d559581a66c493eb8b82f736aa82dfa 2024-12-03T02:20:06,148 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/a/31a6f1ea2e6d414ab240c089637559ec as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/a/31a6f1ea2e6d414ab240c089637559ec 2024-12-03T02:20:06,155 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/a/31a6f1ea2e6d414ab240c089637559ec, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-03T02:20:06,157 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/b/0d559581a66c493eb8b82f736aa82dfa as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/b/0d559581a66c493eb8b82f736aa82dfa 2024-12-03T02:20:06,173 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/b/0d559581a66c493eb8b82f736aa82dfa, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-03T02:20:06,173 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for a3af9ffcc40e129486f79112d9915f21 in 119ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-03T02:20:06,187 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T02:20:06,187 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a3af9ffcc40e129486f79112d9915f21 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T02:20:06,214 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/b/c56d2cdd2c9b49cf83c9de48c03d24a0 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733192401223/Put/seqid=0 2024-12-03T02:20:06,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741861_1038 (size=35082) 2024-12-03T02:20:06,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741861_1038 (size=35082) 2024-12-03T02:20:06,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741861_1038 (size=35082) 2024-12-03T02:20:06,649 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/b/c56d2cdd2c9b49cf83c9de48c03d24a0 2024-12-03T02:20:06,671 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/c/1f5dbd46cb4c4b94b4dfd506029be88a is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733192401257/Put/seqid=0 2024-12-03T02:20:06,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741862_1039 (size=20825) 2024-12-03T02:20:06,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741862_1039 (size=20825) 2024-12-03T02:20:06,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741862_1039 (size=20825) 2024-12-03T02:20:06,681 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/c/1f5dbd46cb4c4b94b4dfd506029be88a 2024-12-03T02:20:06,688 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/b/c56d2cdd2c9b49cf83c9de48c03d24a0 as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/b/c56d2cdd2c9b49cf83c9de48c03d24a0 2024-12-03T02:20:06,695 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/b/c56d2cdd2c9b49cf83c9de48c03d24a0, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-03T02:20:06,697 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/c/1f5dbd46cb4c4b94b4dfd506029be88a as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/c/1f5dbd46cb4c4b94b4dfd506029be88a 2024-12-03T02:20:06,704 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/c/1f5dbd46cb4c4b94b4dfd506029be88a, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-03T02:20:06,704 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for a3af9ffcc40e129486f79112d9915f21 in 517ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-03T02:20:06,711 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733192401304/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733192401035.a3af9ffcc40e129486f79112d9915f21. 2024-12-03T02:20:06,714 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/recovered.edits/0000000000000003002 2024-12-03T02:20:06,714 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T02:20:06,715 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a3af9ffcc40e129486f79112d9915f21 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-03T02:20:06,724 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/c/6e504b57c0814733abbef2f78e92391e is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733192401265/Put/seqid=0 2024-12-03T02:20:06,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741863_1040 (size=50301) 2024-12-03T02:20:06,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741863_1040 (size=50301) 2024-12-03T02:20:06,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741863_1040 (size=50301) 2024-12-03T02:20:06,734 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/c/6e504b57c0814733abbef2f78e92391e 2024-12-03T02:20:06,740 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6e504b57c0814733abbef2f78e92391e 2024-12-03T02:20:06,742 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/.tmp/c/6e504b57c0814733abbef2f78e92391e as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/c/6e504b57c0814733abbef2f78e92391e 2024-12-03T02:20:06,749 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6e504b57c0814733abbef2f78e92391e 2024-12-03T02:20:06,749 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/c/6e504b57c0814733abbef2f78e92391e, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-03T02:20:06,750 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for a3af9ffcc40e129486f79112d9915f21 in 35ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-03T02:20:06,751 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/recovered.edits/0000000000000003002 2024-12-03T02:20:06,752 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:06,752 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:06,753 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T02:20:06,755 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a3af9ffcc40e129486f79112d9915f21 2024-12-03T02:20:06,762 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenIntoWAL/a3af9ffcc40e129486f79112d9915f21/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-03T02:20:06,764 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a3af9ffcc40e129486f79112d9915f21; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62544119, jitterRate=-0.06802000105381012}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T02:20:06,764 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a3af9ffcc40e129486f79112d9915f21: Writing region info on filesystem at 1733192405501Initializing all the Stores at 1733192405502 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192405502Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192405502Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192405502Cleaning up temporary data from old regions at 1733192406752 (+1250 ms)Region opened successfully at 1733192406764 (+12 ms) 2024-12-03T02:20:06,818 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a3af9ffcc40e129486f79112d9915f21, disabling compactions & flushes 2024-12-03T02:20:06,818 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733192401035.a3af9ffcc40e129486f79112d9915f21. 2024-12-03T02:20:06,818 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733192401035.a3af9ffcc40e129486f79112d9915f21. 2024-12-03T02:20:06,818 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733192401035.a3af9ffcc40e129486f79112d9915f21. after waiting 0 ms 2024-12-03T02:20:06,818 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733192401035.a3af9ffcc40e129486f79112d9915f21. 2024-12-03T02:20:06,825 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733192401035.a3af9ffcc40e129486f79112d9915f21. 2024-12-03T02:20:06,825 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a3af9ffcc40e129486f79112d9915f21: Waiting for close lock at 1733192406818Disabling compacts and flushes for region at 1733192406818Disabling writes for close at 1733192406818Writing region close event to WAL at 1733192406824 (+6 ms)Closed at 1733192406824 2024-12-03T02:20:06,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741857_1034 (size=93) 2024-12-03T02:20:06,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741857_1034 (size=93) 2024-12-03T02:20:06,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741857_1034 (size=93) 2024-12-03T02:20:06,831 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T02:20:06,831 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733192405473) 2024-12-03T02:20:06,844 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=404 (was 390) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1779551006_22 at /127.0.0.1:47524 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1162761534) connection to localhost/127.0.0.1:44855 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1162761534) connection to localhost/127.0.0.1:41795 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41795 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1779551006_22 at /127.0.0.1:45862 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1779551006_22 at /127.0.0.1:38032 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34205 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1779551006_22 at /127.0.0.1:38048 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:44855 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1162761534) connection to localhost/127.0.0.1:34205 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=397 (was 415), ProcessCount=11 (was 11), AvailableMemoryMB=292 (was 447) 2024-12-03T02:20:06,857 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=404, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=397, ProcessCount=11, AvailableMemoryMB=292 2024-12-03T02:20:06,874 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:06,879 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:06,881 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:20:06,885 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23514477, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/hregion-23514477, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:06,899 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23514477/hregion-23514477.1733192406885, exclude list is [], retry=0 2024-12-03T02:20:06,904 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:06,905 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:06,906 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:06,908 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23514477/hregion-23514477.1733192406885 2024-12-03T02:20:06,909 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:39215:39215)] 2024-12-03T02:20:06,909 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 82afa2320f82a66ffc855b356c004856, NAME => 'test2727,,1733192406875.82afa2320f82a66ffc855b356c004856.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44855/hbase 2024-12-03T02:20:06,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741865_1042 (size=43) 2024-12-03T02:20:06,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741865_1042 (size=43) 2024-12-03T02:20:06,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741865_1042 (size=43) 2024-12-03T02:20:06,921 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733192406875.82afa2320f82a66ffc855b356c004856.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:06,922 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:06,924 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82afa2320f82a66ffc855b356c004856 columnFamilyName a 2024-12-03T02:20:06,924 DEBUG [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:06,925 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] regionserver.HStore(327): Store=82afa2320f82a66ffc855b356c004856/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:06,926 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:06,932 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82afa2320f82a66ffc855b356c004856 columnFamilyName b 2024-12-03T02:20:06,932 DEBUG [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:06,933 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] regionserver.HStore(327): Store=82afa2320f82a66ffc855b356c004856/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:06,933 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:06,936 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82afa2320f82a66ffc855b356c004856 columnFamilyName c 2024-12-03T02:20:06,936 DEBUG [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:06,937 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] regionserver.HStore(327): Store=82afa2320f82a66ffc855b356c004856/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:06,937 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:06,938 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:06,939 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:06,942 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:06,942 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:06,943 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:06,944 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:06,947 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:20:06,948 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 82afa2320f82a66ffc855b356c004856; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59852300, jitterRate=-0.10813122987747192}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:06,949 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 82afa2320f82a66ffc855b356c004856: Writing region info on filesystem at 1733192406921Initializing all the Stores at 1733192406922 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192406922Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192406922Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192406922Cleaning up temporary data from old regions at 1733192406942 (+20 ms)Region opened successfully at 1733192406949 (+7 ms) 2024-12-03T02:20:06,949 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 82afa2320f82a66ffc855b356c004856, disabling compactions & flushes 2024-12-03T02:20:06,949 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733192406875.82afa2320f82a66ffc855b356c004856. 2024-12-03T02:20:06,949 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733192406875.82afa2320f82a66ffc855b356c004856. 2024-12-03T02:20:06,949 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733192406875.82afa2320f82a66ffc855b356c004856. after waiting 0 ms 2024-12-03T02:20:06,949 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733192406875.82afa2320f82a66ffc855b356c004856. 2024-12-03T02:20:06,950 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733192406875.82afa2320f82a66ffc855b356c004856. 2024-12-03T02:20:06,950 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 82afa2320f82a66ffc855b356c004856: Waiting for close lock at 1733192406949Disabling compacts and flushes for region at 1733192406949Disabling writes for close at 1733192406949Writing region close event to WAL at 1733192406950 (+1 ms)Closed at 1733192406950 2024-12-03T02:20:06,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741864_1041 (size=93) 2024-12-03T02:20:06,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741864_1041 (size=93) 2024-12-03T02:20:06,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741864_1041 (size=93) 2024-12-03T02:20:06,960 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T02:20:06,960 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23514477:(num 1733192406885) 2024-12-03T02:20:06,961 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:06,966 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:06,982 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192406967, exclude list is [], retry=0 2024-12-03T02:20:06,986 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:06,986 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:06,987 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:06,991 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192406967 2024-12-03T02:20:06,999 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:43847:43847)] 2024-12-03T02:20:07,043 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T02:20:07,043 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T02:20:07,045 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T02:20:07,045 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T02:20:07,046 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T02:20:07,046 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T02:20:07,047 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-03T02:20:07,047 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-03T02:20:07,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741866_1043 (size=344306) 2024-12-03T02:20:07,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741866_1043 (size=344306) 2024-12-03T02:20:07,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741866_1043 (size=344306) 2024-12-03T02:20:07,189 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192406967, size=336.2 K (344306bytes) 2024-12-03T02:20:07,189 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192406967 2024-12-03T02:20:07,190 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192406967 after 0ms 2024-12-03T02:20:07,204 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192406967 took 15ms 2024-12-03T02:20:07,230 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733192406967.temp 2024-12-03T02:20:07,233 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000000001-wal.1733192406967.temp 2024-12-03T02:20:07,250 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192406967 so closing down 2024-12-03T02:20:07,250 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T02:20:07,254 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T02:20:07,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741867_1044 (size=344306) 2024-12-03T02:20:07,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741867_1044 (size=344306) 2024-12-03T02:20:07,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741867_1044 (size=344306) 2024-12-03T02:20:07,275 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000000001-wal.1733192406967.temp (wrote 3000 edits, skipped 0 edits in 10 ms) 2024-12-03T02:20:07,276 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000000001-wal.1733192406967.temp to hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000003000 2024-12-03T02:20:07,277 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 62 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192406967, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-12-03T02:20:07,277 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192406967, journal: Splitting hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192406967, size=336.2 K (344306bytes) at 1733192407189Split 1024 edits, skipped 0 edits. at 1733192407229 (+40 ms)Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000000001-wal.1733192406967.temp at 1733192407233 (+4 ms)Split 2048 edits, skipped 0 edits. at 1733192407240 (+7 ms)Finishing writing output for hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192406967 so closing down at 1733192407250 (+10 ms)3 split writer threads finished at 1733192407254 (+4 ms)Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000000001-wal.1733192406967.temp (wrote 3000 edits, skipped 0 edits in 10 ms) at 1733192407275 (+21 ms)Rename recovered edits hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000000001-wal.1733192406967.temp to hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000003000 at 1733192407277 (+2 ms)Processed 3000 edits across 1 Regions in 62 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192406967, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1733192407277 2024-12-03T02:20:07,279 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192406967 to hdfs://localhost:44855/hbase/oldWALs/wal.1733192406967 2024-12-03T02:20:07,280 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000003000 2024-12-03T02:20:07,280 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:07,282 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:07,299 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192407283, exclude list is [], retry=0 2024-12-03T02:20:07,306 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:07,307 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:07,307 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:07,314 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192407283 2024-12-03T02:20:07,315 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:39215:39215)] 2024-12-03T02:20:07,526 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192407283 not finished, retry = 0 2024-12-03T02:20:07,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741868_1045 (size=344433) 2024-12-03T02:20:07,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741868_1045 (size=344433) 2024-12-03T02:20:07,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741868_1045 (size=344433) 2024-12-03T02:20:07,652 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192407283, size=336.4 K (344433bytes) 2024-12-03T02:20:07,652 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192407283 2024-12-03T02:20:07,653 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192407283 after 0ms 2024-12-03T02:20:07,657 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192407283 took 6ms 2024-12-03T02:20:07,662 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733192407283.temp 2024-12-03T02:20:07,664 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000003001-wal.1733192407283.temp 2024-12-03T02:20:07,687 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192407283 so closing down 2024-12-03T02:20:07,687 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T02:20:07,691 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T02:20:07,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741869_1046 (size=344433) 2024-12-03T02:20:07,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741869_1046 (size=344433) 2024-12-03T02:20:07,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741869_1046 (size=344433) 2024-12-03T02:20:07,707 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000003001-wal.1733192407283.temp (wrote 3000 edits, skipped 0 edits in 9 ms) 2024-12-03T02:20:07,709 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000003001-wal.1733192407283.temp to hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000006000 2024-12-03T02:20:07,710 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192407283, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-12-03T02:20:07,710 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192407283, journal: Splitting hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192407283, size=336.4 K (344433bytes) at 1733192407652Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000003001-wal.1733192407283.temp at 1733192407664 (+12 ms)Split 1024 edits, skipped 0 edits. at 1733192407668 (+4 ms)Split 2048 edits, skipped 0 edits. at 1733192407678 (+10 ms)Finishing writing output for hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192407283 so closing down at 1733192407687 (+9 ms)3 split writer threads finished at 1733192407691 (+4 ms)Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000003001-wal.1733192407283.temp (wrote 3000 edits, skipped 0 edits in 9 ms) at 1733192407707 (+16 ms)Rename recovered edits hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000003001-wal.1733192407283.temp to hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000006000 at 1733192407709 (+2 ms)Processed 3000 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192407283, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1733192407710 (+1 ms) 2024-12-03T02:20:07,712 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192407283 to hdfs://localhost:44855/hbase/oldWALs/wal.1733192407283 2024-12-03T02:20:07,714 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000006000 2024-12-03T02:20:07,715 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:07,718 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/test2727-manual,16010,1733192406871, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:07,734 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192407719, exclude list is [], retry=0 2024-12-03T02:20:07,737 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:07,737 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:07,738 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:07,751 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733192406871/wal.1733192407719 2024-12-03T02:20:07,751 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:39215:39215)] 2024-12-03T02:20:07,751 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 82afa2320f82a66ffc855b356c004856, NAME => 'test2727,,1733192406875.82afa2320f82a66ffc855b356c004856.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:07,752 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733192406875.82afa2320f82a66ffc855b356c004856.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:07,752 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:07,752 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:07,755 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:07,757 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82afa2320f82a66ffc855b356c004856 columnFamilyName a 2024-12-03T02:20:07,757 DEBUG [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:07,758 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] regionserver.HStore(327): Store=82afa2320f82a66ffc855b356c004856/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:07,758 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:07,763 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82afa2320f82a66ffc855b356c004856 columnFamilyName b 2024-12-03T02:20:07,763 DEBUG [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:07,764 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] regionserver.HStore(327): Store=82afa2320f82a66ffc855b356c004856/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:07,765 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:07,766 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82afa2320f82a66ffc855b356c004856 columnFamilyName c 2024-12-03T02:20:07,766 DEBUG [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:07,767 INFO [StoreOpener-82afa2320f82a66ffc855b356c004856-1 {}] regionserver.HStore(327): Store=82afa2320f82a66ffc855b356c004856/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:07,767 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:07,768 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:07,771 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:07,772 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000003000 2024-12-03T02:20:07,808 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000003000 2024-12-03T02:20:07,816 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000006000 2024-12-03T02:20:07,862 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000006000 2024-12-03T02:20:07,862 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 82afa2320f82a66ffc855b356c004856 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-03T02:20:07,894 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/.tmp/a/eb81790165c24c86ab2349d14371c5f6 is 41, key is test2727/a:100/1733192407322/Put/seqid=0 2024-12-03T02:20:07,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741871_1048 (size=84227) 2024-12-03T02:20:07,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741871_1048 (size=84227) 2024-12-03T02:20:07,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741871_1048 (size=84227) 2024-12-03T02:20:07,928 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/.tmp/a/eb81790165c24c86ab2349d14371c5f6 2024-12-03T02:20:07,961 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/.tmp/b/07e2b240ca534250b8a7bca905ad68cc is 41, key is test2727/b:100/1733192407394/Put/seqid=0 2024-12-03T02:20:07,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741872_1049 (size=84609) 2024-12-03T02:20:07,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741872_1049 (size=84609) 2024-12-03T02:20:07,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741872_1049 (size=84609) 2024-12-03T02:20:07,971 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/.tmp/b/07e2b240ca534250b8a7bca905ad68cc 2024-12-03T02:20:07,996 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/.tmp/c/426818e6f2de41efb940614625a7508b is 41, key is test2727/c:100/1733192407462/Put/seqid=0 2024-12-03T02:20:08,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741873_1050 (size=84609) 2024-12-03T02:20:08,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741873_1050 (size=84609) 2024-12-03T02:20:08,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741873_1050 (size=84609) 2024-12-03T02:20:08,003 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/.tmp/c/426818e6f2de41efb940614625a7508b 2024-12-03T02:20:08,011 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/.tmp/a/eb81790165c24c86ab2349d14371c5f6 as hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/a/eb81790165c24c86ab2349d14371c5f6 2024-12-03T02:20:08,018 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/a/eb81790165c24c86ab2349d14371c5f6, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-03T02:20:08,020 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/.tmp/b/07e2b240ca534250b8a7bca905ad68cc as hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/b/07e2b240ca534250b8a7bca905ad68cc 2024-12-03T02:20:08,027 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/b/07e2b240ca534250b8a7bca905ad68cc, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-03T02:20:08,028 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/.tmp/c/426818e6f2de41efb940614625a7508b as hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/c/426818e6f2de41efb940614625a7508b 2024-12-03T02:20:08,035 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/c/426818e6f2de41efb940614625a7508b, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-03T02:20:08,036 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 82afa2320f82a66ffc855b356c004856 in 174ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-03T02:20:08,037 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000003000 2024-12-03T02:20:08,037 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/0000000000000006000 2024-12-03T02:20:08,038 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:08,038 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:08,039 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:08,040 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 82afa2320f82a66ffc855b356c004856 2024-12-03T02:20:08,043 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/test2727/82afa2320f82a66ffc855b356c004856/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-03T02:20:08,044 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 82afa2320f82a66ffc855b356c004856; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67458222, jitterRate=0.0052058398723602295}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:08,045 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 82afa2320f82a66ffc855b356c004856: Writing region info on filesystem at 1733192407752Initializing all the Stores at 1733192407754 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192407754Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192407754Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192407754Obtaining lock to block concurrent updates at 1733192407862 (+108 ms)Preparing flush snapshotting stores in 82afa2320f82a66ffc855b356c004856 at 1733192407862Finished memstore snapshotting test2727,,1733192406875.82afa2320f82a66ffc855b356c004856., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733192407863 (+1 ms)Flushing stores of test2727,,1733192406875.82afa2320f82a66ffc855b356c004856. at 1733192407863Flushing 82afa2320f82a66ffc855b356c004856/a: creating writer at 1733192407863Flushing 82afa2320f82a66ffc855b356c004856/a: appending metadata at 1733192407893 (+30 ms)Flushing 82afa2320f82a66ffc855b356c004856/a: closing flushed file at 1733192407893Flushing 82afa2320f82a66ffc855b356c004856/b: creating writer at 1733192407938 (+45 ms)Flushing 82afa2320f82a66ffc855b356c004856/b: appending metadata at 1733192407959 (+21 ms)Flushing 82afa2320f82a66ffc855b356c004856/b: closing flushed file at 1733192407959Flushing 82afa2320f82a66ffc855b356c004856/c: creating writer at 1733192407977 (+18 ms)Flushing 82afa2320f82a66ffc855b356c004856/c: appending metadata at 1733192407996 (+19 ms)Flushing 82afa2320f82a66ffc855b356c004856/c: closing flushed file at 1733192407996Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a62d9d3: reopening flushed file at 1733192408010 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6eca38f7: reopening flushed file at 1733192408019 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e7cf7b7: reopening flushed file at 1733192408027 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 82afa2320f82a66ffc855b356c004856 in 174ms, sequenceid=6000, compaction requested=false; wal=null at 1733192408036 (+9 ms)Cleaning up temporary data from old regions at 1733192408038 (+2 ms)Region opened successfully at 1733192408044 (+6 ms) 2024-12-03T02:20:08,046 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-03T02:20:08,046 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 82afa2320f82a66ffc855b356c004856, disabling compactions & flushes 2024-12-03T02:20:08,046 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733192406875.82afa2320f82a66ffc855b356c004856. 2024-12-03T02:20:08,046 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733192406875.82afa2320f82a66ffc855b356c004856. 2024-12-03T02:20:08,046 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733192406875.82afa2320f82a66ffc855b356c004856. after waiting 0 ms 2024-12-03T02:20:08,046 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733192406875.82afa2320f82a66ffc855b356c004856. 2024-12-03T02:20:08,048 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733192406875.82afa2320f82a66ffc855b356c004856. 2024-12-03T02:20:08,048 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 82afa2320f82a66ffc855b356c004856: Waiting for close lock at 1733192408046Disabling compacts and flushes for region at 1733192408046Disabling writes for close at 1733192408046Writing region close event to WAL at 1733192408048 (+2 ms)Closed at 1733192408048 2024-12-03T02:20:08,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741870_1047 (size=93) 2024-12-03T02:20:08,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741870_1047 (size=93) 2024-12-03T02:20:08,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741870_1047 (size=93) 2024-12-03T02:20:08,055 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T02:20:08,055 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733192407719) 2024-12-03T02:20:08,068 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=408 (was 404) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:47632 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:45862 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:38032 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=397 (was 397), ProcessCount=11 (was 11), AvailableMemoryMB=229 (was 292) 2024-12-03T02:20:08,084 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=408, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=397, ProcessCount=11, AvailableMemoryMB=227 2024-12-03T02:20:08,101 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:08,108 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733192408100, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:08,109 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733192408109 2024-12-03T02:20:08,125 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733192408100/wal.1733192408109 2024-12-03T02:20:08,126 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:39215:39215)] 2024-12-03T02:20:08,128 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dec38b025597b845de90dbb5e6351fd0, NAME => 'testSequentialEditLogSeqNum,,1733192408102.dec38b025597b845de90dbb5e6351fd0.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:08,128 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733192408102.dec38b025597b845de90dbb5e6351fd0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:08,128 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dec38b025597b845de90dbb5e6351fd0 2024-12-03T02:20:08,128 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dec38b025597b845de90dbb5e6351fd0 2024-12-03T02:20:08,130 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:44855/hbase/data/default/testSequentialEditLogSeqNum/dec38b025597b845de90dbb5e6351fd0 doesn't exist for region: dec38b025597b845de90dbb5e6351fd0 on table testSequentialEditLogSeqNum 2024-12-03T02:20:08,130 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: dec38b025597b845de90dbb5e6351fd0 on table testSequentialEditLogSeqNum 2024-12-03T02:20:08,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741875_1052 (size=62) 2024-12-03T02:20:08,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741875_1052 (size=62) 2024-12-03T02:20:08,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741875_1052 (size=62) 2024-12-03T02:20:08,150 INFO [StoreOpener-dec38b025597b845de90dbb5e6351fd0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dec38b025597b845de90dbb5e6351fd0 2024-12-03T02:20:08,152 INFO [StoreOpener-dec38b025597b845de90dbb5e6351fd0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dec38b025597b845de90dbb5e6351fd0 columnFamilyName a 2024-12-03T02:20:08,152 DEBUG [StoreOpener-dec38b025597b845de90dbb5e6351fd0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:08,152 INFO [StoreOpener-dec38b025597b845de90dbb5e6351fd0-1 {}] regionserver.HStore(327): Store=dec38b025597b845de90dbb5e6351fd0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:08,153 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dec38b025597b845de90dbb5e6351fd0 2024-12-03T02:20:08,153 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testSequentialEditLogSeqNum/dec38b025597b845de90dbb5e6351fd0 2024-12-03T02:20:08,154 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testSequentialEditLogSeqNum/dec38b025597b845de90dbb5e6351fd0 2024-12-03T02:20:08,154 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dec38b025597b845de90dbb5e6351fd0 2024-12-03T02:20:08,154 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dec38b025597b845de90dbb5e6351fd0 2024-12-03T02:20:08,156 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dec38b025597b845de90dbb5e6351fd0 2024-12-03T02:20:08,158 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testSequentialEditLogSeqNum/dec38b025597b845de90dbb5e6351fd0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:20:08,159 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dec38b025597b845de90dbb5e6351fd0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59490422, jitterRate=-0.11352363228797913}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T02:20:08,160 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dec38b025597b845de90dbb5e6351fd0: Writing region info on filesystem at 1733192408129Initializing all the Stores at 1733192408149 (+20 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192408149Cleaning up temporary data from old regions at 1733192408154 (+5 ms)Region opened successfully at 1733192408160 (+6 ms) 2024-12-03T02:20:08,171 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dec38b025597b845de90dbb5e6351fd0 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-03T02:20:08,189 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testSequentialEditLogSeqNum/dec38b025597b845de90dbb5e6351fd0/.tmp/a/3ecc9109b6ae47f3965f75146eaa9c5a is 81, key is testSequentialEditLogSeqNum/a:x0/1733192408160/Put/seqid=0 2024-12-03T02:20:08,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741876_1053 (size=5833) 2024-12-03T02:20:08,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741876_1053 (size=5833) 2024-12-03T02:20:08,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741876_1053 (size=5833) 2024-12-03T02:20:08,197 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testSequentialEditLogSeqNum/dec38b025597b845de90dbb5e6351fd0/.tmp/a/3ecc9109b6ae47f3965f75146eaa9c5a 2024-12-03T02:20:08,203 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testSequentialEditLogSeqNum/dec38b025597b845de90dbb5e6351fd0/.tmp/a/3ecc9109b6ae47f3965f75146eaa9c5a as hdfs://localhost:44855/hbase/data/default/testSequentialEditLogSeqNum/dec38b025597b845de90dbb5e6351fd0/a/3ecc9109b6ae47f3965f75146eaa9c5a 2024-12-03T02:20:08,209 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testSequentialEditLogSeqNum/dec38b025597b845de90dbb5e6351fd0/a/3ecc9109b6ae47f3965f75146eaa9c5a, entries=10, sequenceid=13, filesize=5.7 K 2024-12-03T02:20:08,210 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for dec38b025597b845de90dbb5e6351fd0 in 39ms, sequenceid=13, compaction requested=false 2024-12-03T02:20:08,211 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for dec38b025597b845de90dbb5e6351fd0: 2024-12-03T02:20:08,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T02:20:08,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T02:20:08,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T02:20:08,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T02:20:08,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T02:20:08,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741874_1051 (size=3017) 2024-12-03T02:20:08,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741874_1051 (size=3017) 2024-12-03T02:20:08,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741874_1051 (size=3017) 2024-12-03T02:20:08,238 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44855/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733192408100/wal.1733192408109, size=2.9 K (3017bytes) 2024-12-03T02:20:08,238 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44855/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733192408100/wal.1733192408109 2024-12-03T02:20:08,238 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44855/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733192408100/wal.1733192408109 after 0ms 2024-12-03T02:20:08,241 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44855/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733192408100/wal.1733192408109 took 4ms 2024-12-03T02:20:08,244 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44855/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733192408100/wal.1733192408109 so closing down 2024-12-03T02:20:08,244 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T02:20:08,245 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733192408109.temp 2024-12-03T02:20:08,246 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testSequentialEditLogSeqNum/dec38b025597b845de90dbb5e6351fd0/recovered.edits/0000000000000000003-wal.1733192408109.temp 2024-12-03T02:20:08,247 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T02:20:08,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741877_1054 (size=2433) 2024-12-03T02:20:08,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741877_1054 (size=2433) 2024-12-03T02:20:08,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741877_1054 (size=2433) 2024-12-03T02:20:08,254 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testSequentialEditLogSeqNum/dec38b025597b845de90dbb5e6351fd0/recovered.edits/0000000000000000003-wal.1733192408109.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-03T02:20:08,255 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44855/hbase/data/default/testSequentialEditLogSeqNum/dec38b025597b845de90dbb5e6351fd0/recovered.edits/0000000000000000003-wal.1733192408109.temp to hdfs://localhost:44855/hbase/data/default/testSequentialEditLogSeqNum/dec38b025597b845de90dbb5e6351fd0/recovered.edits/0000000000000000020 2024-12-03T02:20:08,255 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:44855/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733192408100/wal.1733192408109, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-12-03T02:20:08,255 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44855/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733192408100/wal.1733192408109, journal: Splitting hdfs://localhost:44855/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733192408100/wal.1733192408109, size=2.9 K (3017bytes) at 1733192408238Finishing writing output for hdfs://localhost:44855/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733192408100/wal.1733192408109 so closing down at 1733192408244 (+6 ms)Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testSequentialEditLogSeqNum/dec38b025597b845de90dbb5e6351fd0/recovered.edits/0000000000000000003-wal.1733192408109.temp at 1733192408246 (+2 ms)3 split writer threads finished at 1733192408247 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testSequentialEditLogSeqNum/dec38b025597b845de90dbb5e6351fd0/recovered.edits/0000000000000000003-wal.1733192408109.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733192408254 (+7 ms)Rename recovered edits hdfs://localhost:44855/hbase/data/default/testSequentialEditLogSeqNum/dec38b025597b845de90dbb5e6351fd0/recovered.edits/0000000000000000003-wal.1733192408109.temp to hdfs://localhost:44855/hbase/data/default/testSequentialEditLogSeqNum/dec38b025597b845de90dbb5e6351fd0/recovered.edits/0000000000000000020 at 1733192408255 (+1 ms)Processed 17 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:44855/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733192408100/wal.1733192408109, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1733192408255 2024-12-03T02:20:08,269 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=413 (was 408) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:47632 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:45862 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=397 (was 397), ProcessCount=11 (was 11), AvailableMemoryMB=251 (was 227) - AvailableMemoryMB LEAK? - 2024-12-03T02:20:08,280 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=413, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=397, ProcessCount=11, AvailableMemoryMB=250 2024-12-03T02:20:08,295 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:08,297 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:08,344 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:20:08,350 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-19701120, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/hregion-19701120, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:08,369 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-19701120/hregion-19701120.1733192408351, exclude list is [], retry=0 2024-12-03T02:20:08,372 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:08,373 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:08,373 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:08,377 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-19701120/hregion-19701120.1733192408351 2024-12-03T02:20:08,377 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:39215:39215)] 2024-12-03T02:20:08,377 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 89c308aca7a300dd67b32a19a3af74a1, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44855/hbase 2024-12-03T02:20:08,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741879_1056 (size=70) 2024-12-03T02:20:08,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741879_1056 (size=70) 2024-12-03T02:20:08,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741879_1056 (size=70) 2024-12-03T02:20:08,396 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:08,397 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,399 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89c308aca7a300dd67b32a19a3af74a1 columnFamilyName a 2024-12-03T02:20:08,399 DEBUG [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:08,400 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(327): Store=89c308aca7a300dd67b32a19a3af74a1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:08,401 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,403 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89c308aca7a300dd67b32a19a3af74a1 columnFamilyName b 2024-12-03T02:20:08,404 DEBUG [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:08,405 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(327): Store=89c308aca7a300dd67b32a19a3af74a1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:08,405 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,407 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89c308aca7a300dd67b32a19a3af74a1 columnFamilyName c 2024-12-03T02:20:08,408 DEBUG [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:08,409 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(327): Store=89c308aca7a300dd67b32a19a3af74a1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:08,409 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,410 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,410 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,412 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,412 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,416 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:08,418 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,422 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:20:08,423 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 89c308aca7a300dd67b32a19a3af74a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63996519, jitterRate=-0.046377554535865784}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:08,425 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 89c308aca7a300dd67b32a19a3af74a1: Writing region info on filesystem at 1733192408396Initializing all the Stores at 1733192408397 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192408397Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192408397Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192408397Cleaning up temporary data from old regions at 1733192408412 (+15 ms)Region opened successfully at 1733192408425 (+13 ms) 2024-12-03T02:20:08,425 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 89c308aca7a300dd67b32a19a3af74a1, disabling compactions & flushes 2024-12-03T02:20:08,425 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1. 2024-12-03T02:20:08,425 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1. 2024-12-03T02:20:08,425 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1. after waiting 0 ms 2024-12-03T02:20:08,426 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1. 2024-12-03T02:20:08,426 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1. 2024-12-03T02:20:08,426 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 89c308aca7a300dd67b32a19a3af74a1: Waiting for close lock at 1733192408425Disabling compacts and flushes for region at 1733192408425Disabling writes for close at 1733192408426 (+1 ms)Writing region close event to WAL at 1733192408426Closed at 1733192408426 2024-12-03T02:20:08,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741878_1055 (size=93) 2024-12-03T02:20:08,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741878_1055 (size=93) 2024-12-03T02:20:08,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741878_1055 (size=93) 2024-12-03T02:20:08,433 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T02:20:08,433 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-19701120:(num 1733192408351) 2024-12-03T02:20:08,434 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:08,436 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:08,449 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436, exclude list is [], retry=0 2024-12-03T02:20:08,452 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:08,452 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:08,453 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:08,455 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436 2024-12-03T02:20:08,455 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:43847:43847)] 2024-12-03T02:20:08,455 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 89c308aca7a300dd67b32a19a3af74a1, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:08,455 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:08,456 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,456 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,457 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,458 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89c308aca7a300dd67b32a19a3af74a1 columnFamilyName a 2024-12-03T02:20:08,458 DEBUG [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:08,459 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(327): Store=89c308aca7a300dd67b32a19a3af74a1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:08,459 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,460 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89c308aca7a300dd67b32a19a3af74a1 columnFamilyName b 2024-12-03T02:20:08,460 DEBUG [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:08,461 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(327): Store=89c308aca7a300dd67b32a19a3af74a1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:08,461 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,462 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89c308aca7a300dd67b32a19a3af74a1 columnFamilyName c 2024-12-03T02:20:08,462 DEBUG [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:08,462 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(327): Store=89c308aca7a300dd67b32a19a3af74a1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:08,462 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,463 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,464 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,465 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,465 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,466 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:08,468 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:08,469 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 89c308aca7a300dd67b32a19a3af74a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61465502, jitterRate=-0.0840926468372345}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:08,470 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 89c308aca7a300dd67b32a19a3af74a1: Writing region info on filesystem at 1733192408456Initializing all the Stores at 1733192408457 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192408457Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192408457Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192408457Cleaning up temporary data from old regions at 1733192408465 (+8 ms)Region opened successfully at 1733192408470 (+5 ms) 2024-12-03T02:20:08,474 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733192408473/Put/seqid=0 2024-12-03T02:20:08,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741881_1058 (size=4826) 2024-12-03T02:20:08,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741881_1058 (size=4826) 2024-12-03T02:20:08,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741881_1058 (size=4826) 2024-12-03T02:20:08,481 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44855/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 89c308aca7a300dd67b32a19a3af74a1/a 2024-12-03T02:20:08,490 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-03T02:20:08,490 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T02:20:08,490 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 89c308aca7a300dd67b32a19a3af74a1: 2024-12-03T02:20:08,492 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/a/3b56090d02c145f185029d7f933f3b78_SeqId_3_ 2024-12-03T02:20:08,493 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44855/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 89c308aca7a300dd67b32a19a3af74a1/a as hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/a/3b56090d02c145f185029d7f933f3b78_SeqId_3_ - updating store file list. 2024-12-03T02:20:08,498 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3b56090d02c145f185029d7f933f3b78_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-03T02:20:08,499 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/a/3b56090d02c145f185029d7f933f3b78_SeqId_3_ into 89c308aca7a300dd67b32a19a3af74a1/a 2024-12-03T02:20:08,499 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44855/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 89c308aca7a300dd67b32a19a3af74a1/a (new location: hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/a/3b56090d02c145f185029d7f933f3b78_SeqId_3_) 2024-12-03T02:20:08,576 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44855/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436, size=0 (0bytes) 2024-12-03T02:20:08,576 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44855/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436 might be still open, length is 0 2024-12-03T02:20:08,577 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44855/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436 2024-12-03T02:20:08,577 WARN [IPC Server handler 1 on default port 44855 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-03T02:20:08,577 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44855/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436 after 0ms 2024-12-03T02:20:10,771 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:46028 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:35125:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46028 dst: /127.0.0.1:35125 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35125 remote=/127.0.0.1:46028]. Total timeout mills is 60000, 57735 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:10,772 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:38216 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36885:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38216 dst: /127.0.0.1:36885 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:10,772 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:47708 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43037:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47708 dst: /127.0.0.1:43037 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:10,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741880_1059 (size=504) 2024-12-03T02:20:10,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741880_1059 (size=504) 2024-12-03T02:20:12,578 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44855/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436 after 4001ms 2024-12-03T02:20:12,582 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44855/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436 took 4006ms 2024-12-03T02:20:12,585 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44855/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436; continuing. 2024-12-03T02:20:12,585 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44855/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436 so closing down 2024-12-03T02:20:12,585 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T02:20:12,587 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733192408436.temp 2024-12-03T02:20:12,589 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/recovered.edits/0000000000000000005-wal.1733192408436.temp 2024-12-03T02:20:12,589 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T02:20:12,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741882_1060 (size=238) 2024-12-03T02:20:12,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741882_1060 (size=238) 2024-12-03T02:20:12,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741882_1060 (size=238) 2024-12-03T02:20:12,609 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/recovered.edits/0000000000000000005-wal.1733192408436.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T02:20:12,610 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/recovered.edits/0000000000000000005-wal.1733192408436.temp to hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/recovered.edits/0000000000000000005 2024-12-03T02:20:12,610 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 27 ms; skipped=1; WAL=hdfs://localhost:44855/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T02:20:12,611 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44855/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436, journal: Splitting hdfs://localhost:44855/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436, size=0 (0bytes) at 1733192408576Finishing writing output for hdfs://localhost:44855/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436 so closing down at 1733192412585 (+4009 ms)Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/recovered.edits/0000000000000000005-wal.1733192408436.temp at 1733192412589 (+4 ms)3 split writer threads finished at 1733192412589Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/recovered.edits/0000000000000000005-wal.1733192408436.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733192412609 (+20 ms)Rename recovered edits hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/recovered.edits/0000000000000000005-wal.1733192408436.temp to hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/recovered.edits/0000000000000000005 at 1733192412610 (+1 ms)Processed 2 edits across 1 Regions in 27 ms; skipped=1; WAL=hdfs://localhost:44855/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436, size=0, length=0, corrupted=false, cancelled=false at 1733192412611 (+1 ms) 2024-12-03T02:20:12,614 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44855/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436 to hdfs://localhost:44855/hbase/oldWALs/wal.1733192408436 2024-12-03T02:20:12,624 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/recovered.edits/0000000000000000005 2024-12-03T02:20:12,624 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:12,626 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:12,641 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192412627, exclude list is [], retry=0 2024-12-03T02:20:12,648 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:12,648 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:12,649 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:12,656 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192412627 2024-12-03T02:20:12,657 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:39215:39215)] 2024-12-03T02:20:12,657 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 89c308aca7a300dd67b32a19a3af74a1, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:12,657 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:12,658 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:12,658 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:12,659 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:12,661 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89c308aca7a300dd67b32a19a3af74a1 columnFamilyName a 2024-12-03T02:20:12,661 DEBUG [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:12,669 DEBUG [StoreFileOpener-89c308aca7a300dd67b32a19a3af74a1-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3b56090d02c145f185029d7f933f3b78_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-03T02:20:12,669 DEBUG [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/a/3b56090d02c145f185029d7f933f3b78_SeqId_3_ 2024-12-03T02:20:12,669 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(327): Store=89c308aca7a300dd67b32a19a3af74a1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:12,669 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:12,670 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89c308aca7a300dd67b32a19a3af74a1 columnFamilyName b 2024-12-03T02:20:12,670 DEBUG [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:12,671 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(327): Store=89c308aca7a300dd67b32a19a3af74a1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:12,671 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:12,672 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89c308aca7a300dd67b32a19a3af74a1 columnFamilyName c 2024-12-03T02:20:12,672 DEBUG [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:12,673 INFO [StoreOpener-89c308aca7a300dd67b32a19a3af74a1-1 {}] regionserver.HStore(327): Store=89c308aca7a300dd67b32a19a3af74a1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:12,673 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:12,674 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:12,675 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:12,676 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/recovered.edits/0000000000000000005 2024-12-03T02:20:12,679 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/recovered.edits/0000000000000000005 2024-12-03T02:20:12,679 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 89c308aca7a300dd67b32a19a3af74a1 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-03T02:20:12,701 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/.tmp/a/aff42585be6c414f8405c3fddef9820f is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733192408504/Put/seqid=0 2024-12-03T02:20:12,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741884_1062 (size=5149) 2024-12-03T02:20:12,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741884_1062 (size=5149) 2024-12-03T02:20:12,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741884_1062 (size=5149) 2024-12-03T02:20:12,710 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/.tmp/a/aff42585be6c414f8405c3fddef9820f 2024-12-03T02:20:12,717 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/.tmp/a/aff42585be6c414f8405c3fddef9820f as hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/a/aff42585be6c414f8405c3fddef9820f 2024-12-03T02:20:12,723 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/a/aff42585be6c414f8405c3fddef9820f, entries=1, sequenceid=5, filesize=5.0 K 2024-12-03T02:20:12,723 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 89c308aca7a300dd67b32a19a3af74a1 in 44ms, sequenceid=5, compaction requested=false; wal=null 2024-12-03T02:20:12,724 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/recovered.edits/0000000000000000005 2024-12-03T02:20:12,726 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:12,726 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:12,727 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:12,729 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 89c308aca7a300dd67b32a19a3af74a1 2024-12-03T02:20:12,732 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/89c308aca7a300dd67b32a19a3af74a1/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-03T02:20:12,733 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 89c308aca7a300dd67b32a19a3af74a1; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65379989, jitterRate=-0.025762245059013367}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:12,734 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 89c308aca7a300dd67b32a19a3af74a1: Writing region info on filesystem at 1733192412658Initializing all the Stores at 1733192412659 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192412659Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192412659Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192412659Obtaining lock to block concurrent updates at 1733192412679 (+20 ms)Preparing flush snapshotting stores in 89c308aca7a300dd67b32a19a3af74a1 at 1733192412679Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733192412680 (+1 ms)Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1. at 1733192412680Flushing 89c308aca7a300dd67b32a19a3af74a1/a: creating writer at 1733192412680Flushing 89c308aca7a300dd67b32a19a3af74a1/a: appending metadata at 1733192412700 (+20 ms)Flushing 89c308aca7a300dd67b32a19a3af74a1/a: closing flushed file at 1733192412700Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d695feb: reopening flushed file at 1733192412716 (+16 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 89c308aca7a300dd67b32a19a3af74a1 in 44ms, sequenceid=5, compaction requested=false; wal=null at 1733192412723 (+7 ms)Cleaning up temporary data from old regions at 1733192412726 (+3 ms)Region opened successfully at 1733192412734 (+8 ms) 2024-12-03T02:20:12,739 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 89c308aca7a300dd67b32a19a3af74a1, disabling compactions & flushes 2024-12-03T02:20:12,740 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1. 2024-12-03T02:20:12,740 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1. 2024-12-03T02:20:12,740 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1. after waiting 0 ms 2024-12-03T02:20:12,740 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1. 2024-12-03T02:20:12,741 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733192408296.89c308aca7a300dd67b32a19a3af74a1. 2024-12-03T02:20:12,741 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 89c308aca7a300dd67b32a19a3af74a1: Waiting for close lock at 1733192412739Disabling compacts and flushes for region at 1733192412739Disabling writes for close at 1733192412740 (+1 ms)Writing region close event to WAL at 1733192412741 (+1 ms)Closed at 1733192412741 2024-12-03T02:20:12,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741883_1061 (size=93) 2024-12-03T02:20:12,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741883_1061 (size=93) 2024-12-03T02:20:12,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741883_1061 (size=93) 2024-12-03T02:20:12,748 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T02:20:12,748 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733192412627) 2024-12-03T02:20:12,768 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=417 (was 413) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1162761534) connection to localhost/127.0.0.1:44855 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1194135099_22 at /127.0.0.1:38256 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1194135099_22 at /127.0.0.1:46036 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1194135099_22 at /127.0.0.1:47734 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:44855 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=374 (was 397), ProcessCount=11 (was 11), AvailableMemoryMB=226 (was 250) 2024-12-03T02:20:12,787 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=417, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=374, ProcessCount=11, AvailableMemoryMB=239 2024-12-03T02:20:12,808 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:12,815 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T02:20:12,821 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 64035cbb0b46,33741,1733192396696 2024-12-03T02:20:12,823 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@25f1137e 2024-12-03T02:20:12,825 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T02:20:12,828 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T02:20:12,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T02:20:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-03T02:20:12,848 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T02:20:12,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-03T02:20:12,851 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:12,853 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T02:20:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T02:20:12,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741885_1063 (size=694) 2024-12-03T02:20:12,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741885_1063 (size=694) 2024-12-03T02:20:12,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741885_1063 (size=694) 2024-12-03T02:20:12,877 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 63293c0fcf6d2a20b48444c1ca6e401a, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c 2024-12-03T02:20:12,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741886_1064 (size=77) 2024-12-03T02:20:12,892 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:12,892 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 63293c0fcf6d2a20b48444c1ca6e401a, disabling compactions & flushes 2024-12-03T02:20:12,893 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:12,893 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:12,893 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. after waiting 0 ms 2024-12-03T02:20:12,893 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:12,893 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:12,893 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 63293c0fcf6d2a20b48444c1ca6e401a: Waiting for close lock at 1733192412892Disabling compacts and flushes for region at 1733192412892Disabling writes for close at 1733192412893 (+1 ms)Writing region close event to WAL at 1733192412893Closed at 1733192412893 2024-12-03T02:20:12,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741886_1064 (size=77) 2024-12-03T02:20:12,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741886_1064 (size=77) 2024-12-03T02:20:12,896 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T02:20:12,902 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733192412897"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733192412897"}]},"ts":"1733192412897"} 2024-12-03T02:20:12,909 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T02:20:12,911 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T02:20:12,915 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733192412912"}]},"ts":"1733192412912"} 2024-12-03T02:20:12,925 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-03T02:20:12,926 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {64035cbb0b46=0} racks are {/default-rack=0} 2024-12-03T02:20:12,932 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T02:20:12,932 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T02:20:12,932 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T02:20:12,932 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T02:20:12,932 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T02:20:12,932 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T02:20:12,932 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T02:20:12,932 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T02:20:12,932 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T02:20:12,932 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T02:20:12,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, ASSIGN}] 2024-12-03T02:20:12,936 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, ASSIGN 2024-12-03T02:20:12,937 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, ASSIGN; state=OFFLINE, location=64035cbb0b46,34333,1733192397467; forceNewPlan=false, retain=false 2024-12-03T02:20:12,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T02:20:13,091 INFO [64035cbb0b46:33741 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T02:20:13,092 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=63293c0fcf6d2a20b48444c1ca6e401a, regionState=OPENING, regionLocation=64035cbb0b46,34333,1733192397467 2024-12-03T02:20:13,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, ASSIGN because future has completed 2024-12-03T02:20:13,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,34333,1733192397467}] 2024-12-03T02:20:13,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T02:20:13,250 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T02:20:13,252 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48209, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T02:20:13,257 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:13,257 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 63293c0fcf6d2a20b48444c1ca6e401a, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:13,258 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:13,258 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:13,258 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:13,258 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:13,260 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:13,261 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63293c0fcf6d2a20b48444c1ca6e401a columnFamilyName cf1 2024-12-03T02:20:13,261 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:13,262 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(327): Store=63293c0fcf6d2a20b48444c1ca6e401a/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:13,262 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:13,264 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63293c0fcf6d2a20b48444c1ca6e401a columnFamilyName cf2 2024-12-03T02:20:13,264 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:13,265 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(327): Store=63293c0fcf6d2a20b48444c1ca6e401a/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:13,265 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:13,266 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:13,267 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:13,268 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:13,268 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:13,269 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T02:20:13,271 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:13,273 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:20:13,274 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 63293c0fcf6d2a20b48444c1ca6e401a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59119592, jitterRate=-0.11904942989349365}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T02:20:13,274 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:13,275 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 63293c0fcf6d2a20b48444c1ca6e401a: Running coprocessor pre-open hook at 1733192413258Writing region info on filesystem at 1733192413258Initializing all the Stores at 1733192413259 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192413259Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192413259Cleaning up temporary data from old regions at 1733192413268 (+9 ms)Running coprocessor post-open hooks at 1733192413274 (+6 ms)Region opened successfully at 1733192413275 (+1 ms) 2024-12-03T02:20:13,277 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., pid=6, masterSystemTime=1733192413250 2024-12-03T02:20:13,280 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:13,281 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:13,281 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=63293c0fcf6d2a20b48444c1ca6e401a, regionState=OPEN, openSeqNum=2, regionLocation=64035cbb0b46,34333,1733192397467 2024-12-03T02:20:13,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,34333,1733192397467 because future has completed 2024-12-03T02:20:13,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T02:20:13,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,34333,1733192397467 in 188 msec 2024-12-03T02:20:13,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T02:20:13,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, ASSIGN in 354 msec 2024-12-03T02:20:13,292 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T02:20:13,293 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733192413292"}]},"ts":"1733192413292"} 2024-12-03T02:20:13,295 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-03T02:20:13,296 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T02:20:13,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 456 msec 2024-12-03T02:20:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T02:20:13,492 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-03T02:20:13,492 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-03T02:20:13,494 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T02:20:13,502 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-03T02:20:13,503 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T02:20:13,504 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-03T02:20:13,517 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=2] 2024-12-03T02:20:13,518 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:20:13,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:20:13,535 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=63293c0fcf6d2a20b48444c1ca6e401a, source=64035cbb0b46,34333,1733192397467, destination=64035cbb0b46,44467,1733192397576, warming up region on 64035cbb0b46,44467,1733192397576 2024-12-03T02:20:13,537 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=63293c0fcf6d2a20b48444c1ca6e401a, source=64035cbb0b46,34333,1733192397467, destination=64035cbb0b46,44467,1733192397576, running balancer 2024-12-03T02:20:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, REOPEN/MOVE 2024-12-03T02:20:13,539 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, REOPEN/MOVE 2024-12-03T02:20:13,542 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44467 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44467 {}] regionserver.HRegion(7855): Warmup {ENCODED => 63293c0fcf6d2a20b48444c1ca6e401a, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44467 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:13,543 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:13,543 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=63293c0fcf6d2a20b48444c1ca6e401a, regionState=CLOSING, regionLocation=64035cbb0b46,34333,1733192397467 2024-12-03T02:20:13,544 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63293c0fcf6d2a20b48444c1ca6e401a columnFamilyName cf1 2024-12-03T02:20:13,545 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:13,546 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(327): Store=63293c0fcf6d2a20b48444c1ca6e401a/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:13,546 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:13,547 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63293c0fcf6d2a20b48444c1ca6e401a columnFamilyName cf2 2024-12-03T02:20:13,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, REOPEN/MOVE because future has completed 2024-12-03T02:20:13,547 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:13,548 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T02:20:13,548 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(327): Store=63293c0fcf6d2a20b48444c1ca6e401a/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44467 {}] regionserver.HRegion(1722): Closing 63293c0fcf6d2a20b48444c1ca6e401a, disabling compactions & flushes 2024-12-03T02:20:13,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44467 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44467 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:13,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,34333,1733192397467}] 2024-12-03T02:20:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44467 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. after waiting 0 ms 2024-12-03T02:20:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44467 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:13,549 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44467 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44467 {}] regionserver.HRegion(1676): Region close journal for 63293c0fcf6d2a20b48444c1ca6e401a: Waiting for close lock at 1733192413548Disabling compacts and flushes for region at 1733192413548Disabling writes for close at 1733192413548Writing region close event to WAL at 1733192413548Closed at 1733192413548 2024-12-03T02:20:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-03T02:20:13,707 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:13,707 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T02:20:13,708 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 63293c0fcf6d2a20b48444c1ca6e401a, disabling compactions & flushes 2024-12-03T02:20:13,708 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:13,708 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:13,708 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. after waiting 0 ms 2024-12-03T02:20:13,708 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:13,708 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 63293c0fcf6d2a20b48444c1ca6e401a 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-03T02:20:13,724 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/.tmp/cf1/7af3f6bd13df433b9d47144307391154 is 35, key is r1/cf1:q/1733192413521/Put/seqid=0 2024-12-03T02:20:13,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741887_1065 (size=4783) 2024-12-03T02:20:13,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741887_1065 (size=4783) 2024-12-03T02:20:13,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741887_1065 (size=4783) 2024-12-03T02:20:13,731 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/.tmp/cf1/7af3f6bd13df433b9d47144307391154 2024-12-03T02:20:13,739 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/.tmp/cf1/7af3f6bd13df433b9d47144307391154 as hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/7af3f6bd13df433b9d47144307391154 2024-12-03T02:20:13,745 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/7af3f6bd13df433b9d47144307391154, entries=1, sequenceid=5, filesize=4.7 K 2024-12-03T02:20:13,746 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 63293c0fcf6d2a20b48444c1ca6e401a in 38ms, sequenceid=5, compaction requested=false 2024-12-03T02:20:13,747 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-03T02:20:13,752 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-03T02:20:13,754 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:13,755 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 63293c0fcf6d2a20b48444c1ca6e401a: Waiting for close lock at 1733192413708Running coprocessor pre-close hooks at 1733192413708Disabling compacts and flushes for region at 1733192413708Disabling writes for close at 1733192413708Obtaining lock to block concurrent updates at 1733192413708Preparing flush snapshotting stores in 63293c0fcf6d2a20b48444c1ca6e401a at 1733192413708Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733192413708Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. at 1733192413709 (+1 ms)Flushing 63293c0fcf6d2a20b48444c1ca6e401a/cf1: creating writer at 1733192413709Flushing 63293c0fcf6d2a20b48444c1ca6e401a/cf1: appending metadata at 1733192413724 (+15 ms)Flushing 63293c0fcf6d2a20b48444c1ca6e401a/cf1: closing flushed file at 1733192413724Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d5852a8: reopening flushed file at 1733192413738 (+14 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 63293c0fcf6d2a20b48444c1ca6e401a in 38ms, sequenceid=5, compaction requested=false at 1733192413747 (+9 ms)Writing region close event to WAL at 1733192413748 (+1 ms)Running coprocessor post-close hooks at 1733192413753 (+5 ms)Closed at 1733192413754 (+1 ms) 2024-12-03T02:20:13,755 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 63293c0fcf6d2a20b48444c1ca6e401a move to 64035cbb0b46,44467,1733192397576 record at close sequenceid=5 2024-12-03T02:20:13,758 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:13,759 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=63293c0fcf6d2a20b48444c1ca6e401a, regionState=CLOSED 2024-12-03T02:20:13,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,34333,1733192397467 because future has completed 2024-12-03T02:20:13,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T02:20:13,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,34333,1733192397467 in 215 msec 2024-12-03T02:20:13,767 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, REOPEN/MOVE; state=CLOSED, location=64035cbb0b46,44467,1733192397576; forceNewPlan=false, retain=false 2024-12-03T02:20:13,917 INFO [64035cbb0b46:33741 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T02:20:13,918 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=63293c0fcf6d2a20b48444c1ca6e401a, regionState=OPENING, regionLocation=64035cbb0b46,44467,1733192397576 2024-12-03T02:20:13,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, REOPEN/MOVE because future has completed 2024-12-03T02:20:13,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,44467,1733192397576}] 2024-12-03T02:20:14,078 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:14,079 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 63293c0fcf6d2a20b48444c1ca6e401a, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:14,079 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,079 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:14,079 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,079 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,085 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,086 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63293c0fcf6d2a20b48444c1ca6e401a columnFamilyName cf1 2024-12-03T02:20:14,086 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:14,095 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/7af3f6bd13df433b9d47144307391154 2024-12-03T02:20:14,095 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(327): Store=63293c0fcf6d2a20b48444c1ca6e401a/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:14,095 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,096 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63293c0fcf6d2a20b48444c1ca6e401a columnFamilyName cf2 2024-12-03T02:20:14,097 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:14,097 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(327): Store=63293c0fcf6d2a20b48444c1ca6e401a/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:14,098 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,098 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,100 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,101 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,101 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,102 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T02:20:14,105 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,106 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 63293c0fcf6d2a20b48444c1ca6e401a; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72096722, jitterRate=0.07432487607002258}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T02:20:14,106 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,107 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 63293c0fcf6d2a20b48444c1ca6e401a: Running coprocessor pre-open hook at 1733192414079Writing region info on filesystem at 1733192414079Initializing all the Stores at 1733192414082 (+3 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192414082Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192414084 (+2 ms)Cleaning up temporary data from old regions at 1733192414101 (+17 ms)Running coprocessor post-open hooks at 1733192414106 (+5 ms)Region opened successfully at 1733192414107 (+1 ms) 2024-12-03T02:20:14,108 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., pid=9, masterSystemTime=1733192414073 2024-12-03T02:20:14,111 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:14,111 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:14,112 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=63293c0fcf6d2a20b48444c1ca6e401a, regionState=OPEN, openSeqNum=9, regionLocation=64035cbb0b46,44467,1733192397576 2024-12-03T02:20:14,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,44467,1733192397576 because future has completed 2024-12-03T02:20:14,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-03T02:20:14,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,44467,1733192397576 in 195 msec 2024-12-03T02:20:14,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, REOPEN/MOVE in 582 msec 2024-12-03T02:20:14,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T02:20:14,145 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T02:20:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34333 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:36780 deadline: 1733192474149, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=64035cbb0b46 port=44467 startCode=1733192397576. As of locationSeqNum=5. 2024-12-03T02:20:14,155 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=64035cbb0b46 port=44467 startCode=1733192397576. As of locationSeqNum=5. 2024-12-03T02:20:14,156 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=64035cbb0b46 port=44467 startCode=1733192397576. As of locationSeqNum=5. 2024-12-03T02:20:14,156 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,44467,1733192397576, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=64035cbb0b46 port=44467 startCode=1733192397576. As of locationSeqNum=5. 2024-12-03T02:20:14,281 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 63293c0fcf6d2a20b48444c1ca6e401a 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-03T02:20:14,298 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/.tmp/cf1/68d909a75ebd463f88dae3c2783e3a83 is 29, key is r1/cf1:/1733192414272/DeleteFamily/seqid=0 2024-12-03T02:20:14,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741888_1066 (size=4906) 2024-12-03T02:20:14,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741888_1066 (size=4906) 2024-12-03T02:20:14,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741888_1066 (size=4906) 2024-12-03T02:20:14,305 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/.tmp/cf1/68d909a75ebd463f88dae3c2783e3a83 2024-12-03T02:20:14,311 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 68d909a75ebd463f88dae3c2783e3a83 2024-12-03T02:20:14,330 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/.tmp/cf2/2843f92d7c5f4b14bb5da13ebb1a5949 is 29, key is r1/cf2:/1733192414272/DeleteFamily/seqid=0 2024-12-03T02:20:14,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741889_1067 (size=4906) 2024-12-03T02:20:14,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741889_1067 (size=4906) 2024-12-03T02:20:14,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741889_1067 (size=4906) 2024-12-03T02:20:14,337 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/.tmp/cf2/2843f92d7c5f4b14bb5da13ebb1a5949 2024-12-03T02:20:14,344 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2843f92d7c5f4b14bb5da13ebb1a5949 2024-12-03T02:20:14,345 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/.tmp/cf1/68d909a75ebd463f88dae3c2783e3a83 as hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/68d909a75ebd463f88dae3c2783e3a83 2024-12-03T02:20:14,352 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 68d909a75ebd463f88dae3c2783e3a83 2024-12-03T02:20:14,352 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/68d909a75ebd463f88dae3c2783e3a83, entries=1, sequenceid=12, filesize=4.8 K 2024-12-03T02:20:14,353 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/.tmp/cf2/2843f92d7c5f4b14bb5da13ebb1a5949 as hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf2/2843f92d7c5f4b14bb5da13ebb1a5949 2024-12-03T02:20:14,361 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2843f92d7c5f4b14bb5da13ebb1a5949 2024-12-03T02:20:14,361 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf2/2843f92d7c5f4b14bb5da13ebb1a5949, entries=1, sequenceid=12, filesize=4.8 K 2024-12-03T02:20:14,363 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 63293c0fcf6d2a20b48444c1ca6e401a in 82ms, sequenceid=12, compaction requested=false 2024-12-03T02:20:14,363 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 63293c0fcf6d2a20b48444c1ca6e401a: 2024-12-03T02:20:14,366 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T02:20:14,368 DEBUG [Time-limited test {}] regionserver.HStore(1541): 63293c0fcf6d2a20b48444c1ca6e401a/cf1 is initiating major compaction (all files) 2024-12-03T02:20:14,369 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T02:20:14,369 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:20:14,369 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 63293c0fcf6d2a20b48444c1ca6e401a/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:14,370 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/7af3f6bd13df433b9d47144307391154, hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/68d909a75ebd463f88dae3c2783e3a83] into tmpdir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/.tmp, totalSize=9.5 K 2024-12-03T02:20:14,371 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7af3f6bd13df433b9d47144307391154, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733192413521 2024-12-03T02:20:14,372 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 68d909a75ebd463f88dae3c2783e3a83, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-03T02:20:14,384 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 63293c0fcf6d2a20b48444c1ca6e401a#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:20:14,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741890_1068 (size=4626) 2024-12-03T02:20:14,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741890_1068 (size=4626) 2024-12-03T02:20:14,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741890_1068 (size=4626) 2024-12-03T02:20:14,411 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/.tmp/cf1/9b68dbf872434a1691ca9aff080f4808 as hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/9b68dbf872434a1691ca9aff080f4808 2024-12-03T02:20:14,425 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 63293c0fcf6d2a20b48444c1ca6e401a/cf1 of 63293c0fcf6d2a20b48444c1ca6e401a into 9b68dbf872434a1691ca9aff080f4808(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:20:14,425 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 63293c0fcf6d2a20b48444c1ca6e401a: 2024-12-03T02:20:14,425 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-03T02:20:14,426 DEBUG [Time-limited test {}] regionserver.HStore(1541): 63293c0fcf6d2a20b48444c1ca6e401a/cf2 is initiating major compaction (all files) 2024-12-03T02:20:14,426 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T02:20:14,426 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:20:14,426 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 63293c0fcf6d2a20b48444c1ca6e401a/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:14,426 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf2/2843f92d7c5f4b14bb5da13ebb1a5949] into tmpdir=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/.tmp, totalSize=4.8 K 2024-12-03T02:20:14,427 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2843f92d7c5f4b14bb5da13ebb1a5949, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-03T02:20:14,433 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 63293c0fcf6d2a20b48444c1ca6e401a#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:20:14,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741891_1069 (size=4592) 2024-12-03T02:20:14,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741891_1069 (size=4592) 2024-12-03T02:20:14,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741891_1069 (size=4592) 2024-12-03T02:20:14,446 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/.tmp/cf2/15a8b2a31d4e4d31b613ffb96de71a09 as hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf2/15a8b2a31d4e4d31b613ffb96de71a09 2024-12-03T02:20:14,453 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 63293c0fcf6d2a20b48444c1ca6e401a/cf2 of 63293c0fcf6d2a20b48444c1ca6e401a into 15a8b2a31d4e4d31b613ffb96de71a09(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:20:14,453 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 63293c0fcf6d2a20b48444c1ca6e401a: 2024-12-03T02:20:14,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=63293c0fcf6d2a20b48444c1ca6e401a, source=64035cbb0b46,44467,1733192397576, destination=64035cbb0b46,34333,1733192397467, warming up region on 64035cbb0b46,34333,1733192397467 2024-12-03T02:20:14,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=63293c0fcf6d2a20b48444c1ca6e401a, source=64035cbb0b46,44467,1733192397576, destination=64035cbb0b46,34333,1733192397467, running balancer 2024-12-03T02:20:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, REOPEN/MOVE 2024-12-03T02:20:14,459 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, REOPEN/MOVE 2024-12-03T02:20:14,460 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=63293c0fcf6d2a20b48444c1ca6e401a, regionState=CLOSING, regionLocation=64035cbb0b46,44467,1733192397576 2024-12-03T02:20:14,460 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34333 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34333 {}] regionserver.HRegion(7855): Warmup {ENCODED => 63293c0fcf6d2a20b48444c1ca6e401a, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34333 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:14,462 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,463 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63293c0fcf6d2a20b48444c1ca6e401a columnFamilyName cf1 2024-12-03T02:20:14,463 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:14,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, REOPEN/MOVE because future has completed 2024-12-03T02:20:14,464 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T02:20:14,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,44467,1733192397576}] 2024-12-03T02:20:14,470 INFO [StoreFileOpener-63293c0fcf6d2a20b48444c1ca6e401a-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 68d909a75ebd463f88dae3c2783e3a83 2024-12-03T02:20:14,470 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/68d909a75ebd463f88dae3c2783e3a83 2024-12-03T02:20:14,475 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/7af3f6bd13df433b9d47144307391154 2024-12-03T02:20:14,479 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/9b68dbf872434a1691ca9aff080f4808 2024-12-03T02:20:14,479 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(327): Store=63293c0fcf6d2a20b48444c1ca6e401a/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:14,480 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,481 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63293c0fcf6d2a20b48444c1ca6e401a columnFamilyName cf2 2024-12-03T02:20:14,481 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:14,488 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf2/15a8b2a31d4e4d31b613ffb96de71a09 2024-12-03T02:20:14,494 INFO [StoreFileOpener-63293c0fcf6d2a20b48444c1ca6e401a-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2843f92d7c5f4b14bb5da13ebb1a5949 2024-12-03T02:20:14,494 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf2/2843f92d7c5f4b14bb5da13ebb1a5949 2024-12-03T02:20:14,494 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(327): Store=63293c0fcf6d2a20b48444c1ca6e401a/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34333 {}] regionserver.HRegion(1722): Closing 63293c0fcf6d2a20b48444c1ca6e401a, disabling compactions & flushes 2024-12-03T02:20:14,495 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34333 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34333 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34333 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. after waiting 0 ms 2024-12-03T02:20:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34333 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:14,496 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34333 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34333 {}] regionserver.HRegion(1676): Region close journal for 63293c0fcf6d2a20b48444c1ca6e401a: Waiting for close lock at 1733192414494Disabling compacts and flushes for region at 1733192414494Disabling writes for close at 1733192414495 (+1 ms)Writing region close event to WAL at 1733192414496 (+1 ms)Closed at 1733192414496 2024-12-03T02:20:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-03T02:20:14,618 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,618 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T02:20:14,618 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 63293c0fcf6d2a20b48444c1ca6e401a, disabling compactions & flushes 2024-12-03T02:20:14,618 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:14,618 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:14,618 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. after waiting 0 ms 2024-12-03T02:20:14,618 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:14,619 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/7af3f6bd13df433b9d47144307391154, hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/68d909a75ebd463f88dae3c2783e3a83] to archive 2024-12-03T02:20:14,622 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:20:14,626 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/7af3f6bd13df433b9d47144307391154 to hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/7af3f6bd13df433b9d47144307391154 2024-12-03T02:20:14,627 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/68d909a75ebd463f88dae3c2783e3a83 to hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/68d909a75ebd463f88dae3c2783e3a83 2024-12-03T02:20:14,640 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf2/2843f92d7c5f4b14bb5da13ebb1a5949] to archive 2024-12-03T02:20:14,641 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:20:14,643 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf2/2843f92d7c5f4b14bb5da13ebb1a5949 to hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf2/2843f92d7c5f4b14bb5da13ebb1a5949 2024-12-03T02:20:14,650 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-03T02:20:14,651 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:14,651 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 63293c0fcf6d2a20b48444c1ca6e401a: Waiting for close lock at 1733192414618Running coprocessor pre-close hooks at 1733192414618Disabling compacts and flushes for region at 1733192414618Disabling writes for close at 1733192414618Writing region close event to WAL at 1733192414645 (+27 ms)Running coprocessor post-close hooks at 1733192414651 (+6 ms)Closed at 1733192414651 2024-12-03T02:20:14,651 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 63293c0fcf6d2a20b48444c1ca6e401a move to 64035cbb0b46,34333,1733192397467 record at close sequenceid=12 2024-12-03T02:20:14,654 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,655 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=63293c0fcf6d2a20b48444c1ca6e401a, regionState=CLOSED 2024-12-03T02:20:14,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,44467,1733192397576 because future has completed 2024-12-03T02:20:14,668 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-03T02:20:14,668 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,44467,1733192397576 in 196 msec 2024-12-03T02:20:14,669 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, REOPEN/MOVE; state=CLOSED, location=64035cbb0b46,34333,1733192397467; forceNewPlan=false, retain=false 2024-12-03T02:20:14,820 INFO [64035cbb0b46:33741 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T02:20:14,820 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=63293c0fcf6d2a20b48444c1ca6e401a, regionState=OPENING, regionLocation=64035cbb0b46,34333,1733192397467 2024-12-03T02:20:14,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, REOPEN/MOVE because future has completed 2024-12-03T02:20:14,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,34333,1733192397467}] 2024-12-03T02:20:14,979 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:14,980 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 63293c0fcf6d2a20b48444c1ca6e401a, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:14,980 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,980 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:14,980 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,980 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,982 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,983 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63293c0fcf6d2a20b48444c1ca6e401a columnFamilyName cf1 2024-12-03T02:20:14,983 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:14,989 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/9b68dbf872434a1691ca9aff080f4808 2024-12-03T02:20:14,989 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(327): Store=63293c0fcf6d2a20b48444c1ca6e401a/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:14,989 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,990 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63293c0fcf6d2a20b48444c1ca6e401a columnFamilyName cf2 2024-12-03T02:20:14,991 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:14,997 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf2/15a8b2a31d4e4d31b613ffb96de71a09 2024-12-03T02:20:14,997 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(327): Store=63293c0fcf6d2a20b48444c1ca6e401a/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:14,997 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:14,998 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:15,000 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:15,000 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:15,001 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:15,002 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T02:20:15,003 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:15,004 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 63293c0fcf6d2a20b48444c1ca6e401a; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66912157, jitterRate=-0.002931162714958191}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T02:20:15,004 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:15,005 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 63293c0fcf6d2a20b48444c1ca6e401a: Running coprocessor pre-open hook at 1733192414980Writing region info on filesystem at 1733192414980Initializing all the Stores at 1733192414981 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192414981Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192414981Cleaning up temporary data from old regions at 1733192415001 (+20 ms)Running coprocessor post-open hooks at 1733192415004 (+3 ms)Region opened successfully at 1733192415005 (+1 ms) 2024-12-03T02:20:15,006 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., pid=12, masterSystemTime=1733192414976 2024-12-03T02:20:15,008 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:15,008 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:15,009 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=63293c0fcf6d2a20b48444c1ca6e401a, regionState=OPEN, openSeqNum=18, regionLocation=64035cbb0b46,34333,1733192397467 2024-12-03T02:20:15,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,34333,1733192397467 because future has completed 2024-12-03T02:20:15,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-03T02:20:15,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,34333,1733192397467 in 189 msec 2024-12-03T02:20:15,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, REOPEN/MOVE in 558 msec 2024-12-03T02:20:15,060 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T02:20:15,062 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T02:20:15,063 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 64035cbb0b46,34333,1733192397467: testing ***** 2024-12-03T02:20:15,063 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-03T02:20:15,065 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-03T02:20:15,066 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-03T02:20:15,068 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-03T02:20:15,069 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-03T02:20:15,076 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 305862112 }, "NonHeapMemoryUsage": { "committed": 169541632, "init": 7667712, "max": -1, "used": 166926512 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "64035cbb0b46", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 13, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 7, "ProcessCallTime_75th_percentile": 10, "ProcessCallTime_90th_percentile": 11, "ProcessCallTime_95th_percentile": 12, "ProcessCallTime_98th_percentile": 12, "ProcessCallTime_99th_percentile": 12, "ProcessCallTime_99.9th_percentile": 12, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 13, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 4, "TotalCallTime_median": 7, "TotalCallTime_75th_percentile": 10, "TotalCallTime_90th_percentile": 11, "TotalCallTime_95th_percentile": 12, "TotalCallTime_98th_percentile": 12, "TotalCallTime_99th_percentile": 12, "TotalCallTime_99.9th_percentile": 12, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "64035cbb0b46", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:57355", "tag.serverName": "64035cbb0b46,39333,1733192397619", "tag.clusterId": "1be2a799-8e6e-40d2-897e-f78b191efd08", "tag.Context": "regionserver", "tag.Hostname": "64035cbb0b46", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733192397619, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-03T02:20:15,079 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33741 {}] master.MasterRpcServices(700): 64035cbb0b46,34333,1733192397467 reported a fatal error: ***** ABORTING region server 64035cbb0b46,34333,1733192397467: testing ***** 2024-12-03T02:20:15,081 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '64035cbb0b46,34333,1733192397467' ***** 2024-12-03T02:20:15,081 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-03T02:20:15,081 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T02:20:15,081 INFO [RS:0;64035cbb0b46:34333 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-03T02:20:15,081 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T02:20:15,081 INFO [RS:0;64035cbb0b46:34333 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-03T02:20:15,082 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(3091): Received CLOSE for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:15,082 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(956): aborting server 64035cbb0b46,34333,1733192397467 2024-12-03T02:20:15,082 INFO [RS:0;64035cbb0b46:34333 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T02:20:15,082 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 63293c0fcf6d2a20b48444c1ca6e401a, disabling compactions & flushes 2024-12-03T02:20:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44467 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:51124 deadline: 1733192475082, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=64035cbb0b46 port=34333 startCode=1733192397467. As of locationSeqNum=12. 2024-12-03T02:20:15,082 INFO [RS:0;64035cbb0b46:34333 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;64035cbb0b46:34333. 2024-12-03T02:20:15,082 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:15,082 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:15,083 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. after waiting 0 ms 2024-12-03T02:20:15,083 DEBUG [RS:0;64035cbb0b46:34333 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:20:15,083 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:15,083 DEBUG [RS:0;64035cbb0b46:34333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:20:15,083 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,44467,1733192397576, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,44467,1733192397576, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=64035cbb0b46 port=34333 startCode=1733192397467. As of locationSeqNum=12. 2024-12-03T02:20:15,083 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,44467,1733192397576, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=64035cbb0b46 port=34333 startCode=1733192397467. As of locationSeqNum=12. 2024-12-03T02:20:15,083 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T02:20:15,083 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,44467,1733192397576, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=64035cbb0b46 port=34333 startCode=1733192397467. As of locationSeqNum=12. 2024-12-03T02:20:15,083 DEBUG [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(1325): Online Regions={63293c0fcf6d2a20b48444c1ca6e401a=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.} 2024-12-03T02:20:15,084 DEBUG [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(1351): Waiting on 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:15,085 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:15,085 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 63293c0fcf6d2a20b48444c1ca6e401a: Waiting for close lock at 1733192415082Running coprocessor pre-close hooks at 1733192415082Disabling compacts and flushes for region at 1733192415082Disabling writes for close at 1733192415083 (+1 ms)Writing region close event to WAL at 1733192415084 (+1 ms)Running coprocessor post-close hooks at 1733192415084Closed at 1733192415085 (+1 ms) 2024-12-03T02:20:15,085 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:15,119 INFO [regionserver/64035cbb0b46:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T02:20:15,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 64035cbb0b46,34333,1733192397467 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:15,192 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 64035cbb0b46,34333,1733192397467 aborting 2024-12-03T02:20:15,192 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 64035cbb0b46,34333,1733192397467 aborting 2024-12-03T02:20:15,192 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=12 from cache 2024-12-03T02:20:15,284 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(976): stopping server 64035cbb0b46,34333,1733192397467; all regions closed. 2024-12-03T02:20:15,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741835_1011 (size=2142) 2024-12-03T02:20:15,288 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467/64035cbb0b46%2C34333%2C1733192397467.1733192399275 not finished, retry = 0 2024-12-03T02:20:15,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741835_1011 (size=2142) 2024-12-03T02:20:15,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741835_1011 (size=2142) 2024-12-03T02:20:15,389 DEBUG [RS:0;64035cbb0b46:34333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:20:15,390 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T02:20:15,390 INFO [RS:0;64035cbb0b46:34333 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T02:20:15,390 INFO [RS:0;64035cbb0b46:34333 {}] hbase.ChoreService(370): Chore service for: regionserver/64035cbb0b46:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-03T02:20:15,390 INFO [regionserver/64035cbb0b46:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T02:20:15,390 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T02:20:15,390 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T02:20:15,390 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T02:20:15,390 INFO [RS:0;64035cbb0b46:34333 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T02:20:15,391 INFO [RS:0;64035cbb0b46:34333 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34333 2024-12-03T02:20:15,403 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=18] 2024-12-03T02:20:15,404 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 64035cbb0b46:34333 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 64035cbb0b46/172.17.0.3:34333 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:15,405 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=18, error=java.net.ConnectException: Call to address=64035cbb0b46:34333 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 64035cbb0b46/172.17.0.3:34333 2024-12-03T02:20:15,405 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-03T02:20:15,405 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=18 from cache 2024-12-03T02:20:15,405 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 64035cbb0b46:34333 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 64035cbb0b46/172.17.0.3:34333 2024-12-03T02:20:15,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T02:20:15,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34333-0x1019a46de8c0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/64035cbb0b46,34333,1733192397467 2024-12-03T02:20:15,432 INFO [RS:0;64035cbb0b46:34333 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T02:20:15,433 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [64035cbb0b46,34333,1733192397467] 2024-12-03T02:20:15,636 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/64035cbb0b46,34333,1733192397467 already deleted, retry=false 2024-12-03T02:20:15,636 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 64035cbb0b46,34333,1733192397467 on 64035cbb0b46,33741,1733192396696 2024-12-03T02:20:15,642 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 64035cbb0b46,34333,1733192397467, splitWal=true, meta=false 2024-12-03T02:20:15,644 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 64035cbb0b46,34333,1733192397467 (carryingMeta=false) 64035cbb0b46,34333,1733192397467/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@14b49c16[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-03T02:20:15,645 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 64035cbb0b46,34333,1733192397467, splitWal=true, meta=false 2024-12-03T02:20:15,647 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 64035cbb0b46,34333,1733192397467 had 1 regions 2024-12-03T02:20:15,648 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 64035cbb0b46,34333,1733192397467, splitWal=true, meta=false, isMeta: false 2024-12-03T02:20:15,650 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467-splitting 2024-12-03T02:20:15,651 INFO [PEWorker-1 {}] master.SplitWALManager(105): 64035cbb0b46,34333,1733192397467 WAL count=1, meta=false 2024-12-03T02:20:15,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 64035cbb0b46%2C34333%2C1733192397467.1733192399275}] 2024-12-03T02:20:15,659 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=64035cbb0b46,39333,1733192397619 2024-12-03T02:20:15,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 64035cbb0b46%2C34333%2C1733192397467.1733192399275, worker=64035cbb0b46,39333,1733192397619}] 2024-12-03T02:20:15,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34333-0x1019a46de8c0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T02:20:15,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34333-0x1019a46de8c0001, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T02:20:15,666 INFO [RS:0;64035cbb0b46:34333 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T02:20:15,666 INFO [RS:0;64035cbb0b46:34333 {}] regionserver.HRegionServer(1031): Exiting; stopping=64035cbb0b46,34333,1733192397467; zookeeper connection closed. 2024-12-03T02:20:15,666 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7cca5ed6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7cca5ed6 2024-12-03T02:20:15,713 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=18] 2024-12-03T02:20:15,713 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 64035cbb0b46:34333 this server is in the failed servers list 2024-12-03T02:20:15,714 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=64035cbb0b46:34333 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 64035cbb0b46:34333 2024-12-03T02:20:15,714 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 64035cbb0b46:34333 2024-12-03T02:20:15,714 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=18 from cache 2024-12-03T02:20:15,822 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T02:20:15,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58383, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T02:20:15,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39333 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-03T02:20:15,839 INFO [RS_LOG_REPLAY_OPS-regionserver/64035cbb0b46:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467-splitting/64035cbb0b46%2C34333%2C1733192397467.1733192399275, size=2.1 K (2142bytes) 2024-12-03T02:20:15,840 INFO [RS_LOG_REPLAY_OPS-regionserver/64035cbb0b46:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467-splitting/64035cbb0b46%2C34333%2C1733192397467.1733192399275 2024-12-03T02:20:15,840 INFO [RS_LOG_REPLAY_OPS-regionserver/64035cbb0b46:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467-splitting/64035cbb0b46%2C34333%2C1733192397467.1733192399275 after 0ms 2024-12-03T02:20:15,843 INFO [RS_LOG_REPLAY_OPS-regionserver/64035cbb0b46:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467-splitting/64035cbb0b46%2C34333%2C1733192397467.1733192399275 took 4ms 2024-12-03T02:20:15,849 DEBUG [RS_LOG_REPLAY_OPS-regionserver/64035cbb0b46:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 63293c0fcf6d2a20b48444c1ca6e401a: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-03T02:20:15,849 DEBUG [RS_LOG_REPLAY_OPS-regionserver/64035cbb0b46:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467-splitting/64035cbb0b46%2C34333%2C1733192397467.1733192399275 so closing down 2024-12-03T02:20:15,849 DEBUG [RS_LOG_REPLAY_OPS-regionserver/64035cbb0b46:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T02:20:15,849 INFO [RS_LOG_REPLAY_OPS-regionserver/64035cbb0b46:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T02:20:15,849 INFO [RS_LOG_REPLAY_OPS-regionserver/64035cbb0b46:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467-splitting/64035cbb0b46%2C34333%2C1733192397467.1733192399275, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-12-03T02:20:15,849 DEBUG [RS_LOG_REPLAY_OPS-regionserver/64035cbb0b46:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467-splitting/64035cbb0b46%2C34333%2C1733192397467.1733192399275, journal: Splitting hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467-splitting/64035cbb0b46%2C34333%2C1733192397467.1733192399275, size=2.1 K (2142bytes) at 1733192415840Finishing writing output for hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467-splitting/64035cbb0b46%2C34333%2C1733192397467.1733192399275 so closing down at 1733192415849 (+9 ms)3 split writer threads finished at 1733192415849Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467-splitting/64035cbb0b46%2C34333%2C1733192397467.1733192399275, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1733192415849 2024-12-03T02:20:15,849 DEBUG [RS_LOG_REPLAY_OPS-regionserver/64035cbb0b46:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467-splitting/64035cbb0b46%2C34333%2C1733192397467.1733192399275 2024-12-03T02:20:15,850 DEBUG [RS_LOG_REPLAY_OPS-regionserver/64035cbb0b46:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-03T02:20:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33741 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-03T02:20:15,857 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467-splitting/64035cbb0b46%2C34333%2C1733192397467.1733192399275 to hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/oldWALs 2024-12-03T02:20:15,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-03T02:20:15,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 64035cbb0b46%2C34333%2C1733192397467.1733192399275, worker=64035cbb0b46,39333,1733192397619 in 196 msec 2024-12-03T02:20:15,862 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=64035cbb0b46,39333,1733192397619 2024-12-03T02:20:15,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T02:20:15,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 64035cbb0b46%2C34333%2C1733192397467.1733192399275, worker=64035cbb0b46,39333,1733192397619 in 210 msec 2024-12-03T02:20:15,867 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467-splitting dir is empty, no logs to split. 2024-12-03T02:20:15,867 INFO [PEWorker-1 {}] master.SplitWALManager(105): 64035cbb0b46,34333,1733192397467 WAL count=0, meta=false 2024-12-03T02:20:15,867 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 64035cbb0b46,34333,1733192397467 WAL splitting is done? wals=0, meta=false 2024-12-03T02:20:15,869 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 64035cbb0b46,34333,1733192397467 failed, ignore...File hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/WALs/64035cbb0b46,34333,1733192397467-splitting does not exist. 2024-12-03T02:20:15,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, ASSIGN}] 2024-12-03T02:20:15,873 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, ASSIGN 2024-12-03T02:20:15,874 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T02:20:16,025 DEBUG [64035cbb0b46:33741 {}] balancer.BalancerClusterState(204): Hosts are {64035cbb0b46=0} racks are {/default-rack=0} 2024-12-03T02:20:16,025 DEBUG [64035cbb0b46:33741 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T02:20:16,025 DEBUG [64035cbb0b46:33741 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T02:20:16,025 DEBUG [64035cbb0b46:33741 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T02:20:16,025 DEBUG [64035cbb0b46:33741 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T02:20:16,025 INFO [64035cbb0b46:33741 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T02:20:16,025 INFO [64035cbb0b46:33741 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T02:20:16,025 DEBUG [64035cbb0b46:33741 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T02:20:16,025 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=63293c0fcf6d2a20b48444c1ca6e401a, regionState=OPENING, regionLocation=64035cbb0b46,39333,1733192397619 2024-12-03T02:20:16,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, ASSIGN because future has completed 2024-12-03T02:20:16,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,39333,1733192397619}] 2024-12-03T02:20:16,186 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:16,187 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 63293c0fcf6d2a20b48444c1ca6e401a, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:16,187 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:16,187 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:16,188 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:16,188 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:16,190 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:16,191 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63293c0fcf6d2a20b48444c1ca6e401a columnFamilyName cf1 2024-12-03T02:20:16,191 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:16,197 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf1/9b68dbf872434a1691ca9aff080f4808 2024-12-03T02:20:16,198 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(327): Store=63293c0fcf6d2a20b48444c1ca6e401a/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:16,198 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:16,199 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63293c0fcf6d2a20b48444c1ca6e401a columnFamilyName cf2 2024-12-03T02:20:16,199 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:16,206 DEBUG [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/cf2/15a8b2a31d4e4d31b613ffb96de71a09 2024-12-03T02:20:16,206 INFO [StoreOpener-63293c0fcf6d2a20b48444c1ca6e401a-1 {}] regionserver.HStore(327): Store=63293c0fcf6d2a20b48444c1ca6e401a/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:16,206 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:16,207 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:16,209 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:16,210 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:16,210 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:16,211 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T02:20:16,213 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:16,214 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 63293c0fcf6d2a20b48444c1ca6e401a; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69207584, jitterRate=0.03127336502075195}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T02:20:16,214 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:16,214 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 63293c0fcf6d2a20b48444c1ca6e401a: Running coprocessor pre-open hook at 1733192416188Writing region info on filesystem at 1733192416188Initializing all the Stores at 1733192416189 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192416189Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192416189Cleaning up temporary data from old regions at 1733192416210 (+21 ms)Running coprocessor post-open hooks at 1733192416214 (+4 ms)Region opened successfully at 1733192416214 2024-12-03T02:20:16,215 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., pid=17, masterSystemTime=1733192416181 2024-12-03T02:20:16,218 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:16,218 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:16,218 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=63293c0fcf6d2a20b48444c1ca6e401a, regionState=OPEN, openSeqNum=18, regionLocation=64035cbb0b46,39333,1733192397619 2024-12-03T02:20:16,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,39333,1733192397619 because future has completed 2024-12-03T02:20:16,222 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=18] 2024-12-03T02:20:16,223 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 64035cbb0b46:34333 this server is in the failed servers list 2024-12-03T02:20:16,223 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=64035cbb0b46:34333 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 64035cbb0b46:34333 2024-12-03T02:20:16,223 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 64035cbb0b46:34333 2024-12-03T02:20:16,223 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,34333,1733192397467, seqNum=18 from cache 2024-12-03T02:20:16,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-03T02:20:16,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 63293c0fcf6d2a20b48444c1ca6e401a, server=64035cbb0b46,39333,1733192397619 in 196 msec 2024-12-03T02:20:16,229 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-03T02:20:16,229 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=63293c0fcf6d2a20b48444c1ca6e401a, ASSIGN in 354 msec 2024-12-03T02:20:16,229 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 64035cbb0b46,34333,1733192397467 after splitting done 2024-12-03T02:20:16,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 64035cbb0b46,34333,1733192397467, splitWal=true, meta=false in 591 msec 2024-12-03T02:20:17,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-03T02:20:17,043 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-03T02:20:17,044 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-03T02:20:17,044 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-03T02:20:17,044 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-03T02:20:17,045 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-03T02:20:17,242 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a., hostname=64035cbb0b46,39333,1733192397619, seqNum=18] 2024-12-03T02:20:17,242 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:20:17,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:20:17,257 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=411 (was 417), OpenFileDescriptor=1049 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=368 (was 374), ProcessCount=11 (was 11), AvailableMemoryMB=238 (was 239) 2024-12-03T02:20:17,259 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1049 is superior to 1024 2024-12-03T02:20:17,273 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=411, OpenFileDescriptor=1049, MaxFileDescriptor=1048576, SystemLoadAverage=368, ProcessCount=11, AvailableMemoryMB=238 2024-12-03T02:20:17,273 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1049 is superior to 1024 2024-12-03T02:20:17,287 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:17,289 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:17,289 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:20:17,292 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23868887, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/hregion-23868887, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:17,304 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23868887/hregion-23868887.1733192417293, exclude list is [], retry=0 2024-12-03T02:20:17,307 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:17,307 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:17,307 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:17,310 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23868887/hregion-23868887.1733192417293 2024-12-03T02:20:17,310 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:36675:36675)] 2024-12-03T02:20:17,310 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a84ddab9aaac1e3bba10b1e54a42db42, NAME => 'testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44855/hbase 2024-12-03T02:20:17,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741893_1071 (size=67) 2024-12-03T02:20:17,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741893_1071 (size=67) 2024-12-03T02:20:17,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741893_1071 (size=67) 2024-12-03T02:20:17,319 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:17,321 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,323 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84ddab9aaac1e3bba10b1e54a42db42 columnFamilyName a 2024-12-03T02:20:17,323 DEBUG [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:17,324 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(327): Store=a84ddab9aaac1e3bba10b1e54a42db42/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:17,324 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,326 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84ddab9aaac1e3bba10b1e54a42db42 columnFamilyName b 2024-12-03T02:20:17,326 DEBUG [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:17,327 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(327): Store=a84ddab9aaac1e3bba10b1e54a42db42/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:17,327 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,329 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84ddab9aaac1e3bba10b1e54a42db42 columnFamilyName c 2024-12-03T02:20:17,329 DEBUG [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:17,330 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(327): Store=a84ddab9aaac1e3bba10b1e54a42db42/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:17,330 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,331 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,332 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,333 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,333 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,334 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:17,335 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,338 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:20:17,339 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a84ddab9aaac1e3bba10b1e54a42db42; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70810296, jitterRate=0.05515563488006592}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:17,340 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a84ddab9aaac1e3bba10b1e54a42db42: Writing region info on filesystem at 1733192417320Initializing all the Stores at 1733192417321 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192417321Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192417321Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192417321Cleaning up temporary data from old regions at 1733192417333 (+12 ms)Region opened successfully at 1733192417340 (+7 ms) 2024-12-03T02:20:17,340 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a84ddab9aaac1e3bba10b1e54a42db42, disabling compactions & flushes 2024-12-03T02:20:17,340 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42. 2024-12-03T02:20:17,340 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42. 2024-12-03T02:20:17,340 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42. after waiting 0 ms 2024-12-03T02:20:17,340 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42. 2024-12-03T02:20:17,341 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42. 2024-12-03T02:20:17,341 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a84ddab9aaac1e3bba10b1e54a42db42: Waiting for close lock at 1733192417340Disabling compacts and flushes for region at 1733192417340Disabling writes for close at 1733192417340Writing region close event to WAL at 1733192417341 (+1 ms)Closed at 1733192417341 2024-12-03T02:20:17,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741892_1070 (size=93) 2024-12-03T02:20:17,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741892_1070 (size=93) 2024-12-03T02:20:17,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741892_1070 (size=93) 2024-12-03T02:20:17,353 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T02:20:17,353 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23868887:(num 1733192417293) 2024-12-03T02:20:17,353 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:17,356 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:17,368 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417356, exclude list is [], retry=0 2024-12-03T02:20:17,371 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:17,371 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:17,371 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:17,376 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417356 2024-12-03T02:20:17,376 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:43847:43847)] 2024-12-03T02:20:17,376 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a84ddab9aaac1e3bba10b1e54a42db42, NAME => 'testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:17,376 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:17,376 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,376 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,378 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,378 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84ddab9aaac1e3bba10b1e54a42db42 columnFamilyName a 2024-12-03T02:20:17,378 DEBUG [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:17,379 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(327): Store=a84ddab9aaac1e3bba10b1e54a42db42/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:17,379 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,380 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84ddab9aaac1e3bba10b1e54a42db42 columnFamilyName b 2024-12-03T02:20:17,380 DEBUG [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:17,380 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(327): Store=a84ddab9aaac1e3bba10b1e54a42db42/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:17,380 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,381 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84ddab9aaac1e3bba10b1e54a42db42 columnFamilyName c 2024-12-03T02:20:17,381 DEBUG [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:17,382 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(327): Store=a84ddab9aaac1e3bba10b1e54a42db42/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:17,382 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,383 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,384 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,385 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,385 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,386 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:17,387 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,388 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a84ddab9aaac1e3bba10b1e54a42db42; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61022011, jitterRate=-0.09070117771625519}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:17,389 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a84ddab9aaac1e3bba10b1e54a42db42: Writing region info on filesystem at 1733192417377Initializing all the Stores at 1733192417377Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192417377Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192417378 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192417378Cleaning up temporary data from old regions at 1733192417385 (+7 ms)Region opened successfully at 1733192417389 (+4 ms) 2024-12-03T02:20:17,415 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a84ddab9aaac1e3bba10b1e54a42db42 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-03T02:20:17,431 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/.tmp/a/6fa77b65bc4b4aae8cf4121d44112a0c is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733192417389/Put/seqid=0 2024-12-03T02:20:17,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741895_1073 (size=5958) 2024-12-03T02:20:17,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741895_1073 (size=5958) 2024-12-03T02:20:17,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741895_1073 (size=5958) 2024-12-03T02:20:17,442 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/.tmp/a/6fa77b65bc4b4aae8cf4121d44112a0c 2024-12-03T02:20:17,461 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/.tmp/b/300b11717a23494db05dc20189636f4e is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733192417397/Put/seqid=0 2024-12-03T02:20:17,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741896_1074 (size=5958) 2024-12-03T02:20:17,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741896_1074 (size=5958) 2024-12-03T02:20:17,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741896_1074 (size=5958) 2024-12-03T02:20:17,468 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/.tmp/b/300b11717a23494db05dc20189636f4e 2024-12-03T02:20:17,487 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/.tmp/c/ac33d1fc81d34ca29472635302862bd1 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733192417405/Put/seqid=0 2024-12-03T02:20:17,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741897_1075 (size=5958) 2024-12-03T02:20:17,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741897_1075 (size=5958) 2024-12-03T02:20:17,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741897_1075 (size=5958) 2024-12-03T02:20:17,494 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/.tmp/c/ac33d1fc81d34ca29472635302862bd1 2024-12-03T02:20:17,499 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/.tmp/a/6fa77b65bc4b4aae8cf4121d44112a0c as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/a/6fa77b65bc4b4aae8cf4121d44112a0c 2024-12-03T02:20:17,504 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/a/6fa77b65bc4b4aae8cf4121d44112a0c, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T02:20:17,505 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/.tmp/b/300b11717a23494db05dc20189636f4e as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/b/300b11717a23494db05dc20189636f4e 2024-12-03T02:20:17,510 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/b/300b11717a23494db05dc20189636f4e, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T02:20:17,512 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/.tmp/c/ac33d1fc81d34ca29472635302862bd1 as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/c/ac33d1fc81d34ca29472635302862bd1 2024-12-03T02:20:17,517 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/c/ac33d1fc81d34ca29472635302862bd1, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T02:20:17,518 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for a84ddab9aaac1e3bba10b1e54a42db42 in 103ms, sequenceid=33, compaction requested=false 2024-12-03T02:20:17,518 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a84ddab9aaac1e3bba10b1e54a42db42: 2024-12-03T02:20:17,519 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a84ddab9aaac1e3bba10b1e54a42db42, disabling compactions & flushes 2024-12-03T02:20:17,519 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42. 2024-12-03T02:20:17,519 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42. 2024-12-03T02:20:17,519 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42. after waiting 0 ms 2024-12-03T02:20:17,519 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42. 2024-12-03T02:20:17,520 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42. 2024-12-03T02:20:17,520 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a84ddab9aaac1e3bba10b1e54a42db42: Waiting for close lock at 1733192417518Disabling compacts and flushes for region at 1733192417519 (+1 ms)Disabling writes for close at 1733192417519Writing region close event to WAL at 1733192417520 (+1 ms)Closed at 1733192417520 2024-12-03T02:20:17,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741894_1072 (size=5937) 2024-12-03T02:20:17,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741894_1072 (size=5937) 2024-12-03T02:20:17,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741894_1072 (size=5937) 2024-12-03T02:20:17,530 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/b/300b11717a23494db05dc20189636f4e to hdfs://localhost:44855/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/b/300b11717a23494db05dc20189636f4e 2024-12-03T02:20:17,545 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44855/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417356, size=5.8 K (5937bytes) 2024-12-03T02:20:17,546 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44855/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417356 2024-12-03T02:20:17,546 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44855/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417356 after 0ms 2024-12-03T02:20:17,549 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44855/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417356 took 4ms 2024-12-03T02:20:17,551 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44855/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417356 so closing down 2024-12-03T02:20:17,551 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T02:20:17,552 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733192417356.temp 2024-12-03T02:20:17,553 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/recovered.edits/0000000000000000003-wal.1733192417356.temp 2024-12-03T02:20:17,553 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T02:20:17,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741898_1076 (size=5223) 2024-12-03T02:20:17,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741898_1076 (size=5223) 2024-12-03T02:20:17,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741898_1076 (size=5223) 2024-12-03T02:20:17,559 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/recovered.edits/0000000000000000003-wal.1733192417356.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T02:20:17,561 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/recovered.edits/0000000000000000003-wal.1733192417356.temp to hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/recovered.edits/0000000000000000032 2024-12-03T02:20:17,561 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:44855/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417356, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-12-03T02:20:17,561 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44855/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417356, journal: Splitting hdfs://localhost:44855/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417356, size=5.8 K (5937bytes) at 1733192417546Finishing writing output for hdfs://localhost:44855/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417356 so closing down at 1733192417551 (+5 ms)Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/recovered.edits/0000000000000000003-wal.1733192417356.temp at 1733192417553 (+2 ms)3 split writer threads finished at 1733192417553Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/recovered.edits/0000000000000000003-wal.1733192417356.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733192417559 (+6 ms)Rename recovered edits hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/recovered.edits/0000000000000000003-wal.1733192417356.temp to hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/recovered.edits/0000000000000000032 at 1733192417561 (+2 ms)Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:44855/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417356, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1733192417561 2024-12-03T02:20:17,563 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44855/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417356 to hdfs://localhost:44855/hbase/oldWALs/wal.1733192417356 2024-12-03T02:20:17,564 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/recovered.edits/0000000000000000032 2024-12-03T02:20:17,564 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:17,565 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:17,576 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417565, exclude list is [], retry=0 2024-12-03T02:20:17,578 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:17,579 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:17,579 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:17,580 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417565 2024-12-03T02:20:17,581 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:43847:43847)] 2024-12-03T02:20:17,581 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a84ddab9aaac1e3bba10b1e54a42db42, NAME => 'testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:17,581 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:17,581 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,581 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,583 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,583 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84ddab9aaac1e3bba10b1e54a42db42 columnFamilyName a 2024-12-03T02:20:17,583 DEBUG [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:17,589 DEBUG [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/a/6fa77b65bc4b4aae8cf4121d44112a0c 2024-12-03T02:20:17,589 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(327): Store=a84ddab9aaac1e3bba10b1e54a42db42/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:17,589 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,590 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84ddab9aaac1e3bba10b1e54a42db42 columnFamilyName b 2024-12-03T02:20:17,590 DEBUG [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:17,591 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(327): Store=a84ddab9aaac1e3bba10b1e54a42db42/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:17,591 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,592 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84ddab9aaac1e3bba10b1e54a42db42 columnFamilyName c 2024-12-03T02:20:17,592 DEBUG [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:17,597 DEBUG [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/c/ac33d1fc81d34ca29472635302862bd1 2024-12-03T02:20:17,597 INFO [StoreOpener-a84ddab9aaac1e3bba10b1e54a42db42-1 {}] regionserver.HStore(327): Store=a84ddab9aaac1e3bba10b1e54a42db42/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:17,597 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,598 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,599 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,600 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/recovered.edits/0000000000000000032 2024-12-03T02:20:17,603 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/recovered.edits/0000000000000000032 2024-12-03T02:20:17,603 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a84ddab9aaac1e3bba10b1e54a42db42 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-03T02:20:17,615 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/.tmp/b/0c3cef69438d4963a1ad3380eb0d36f0 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733192417397/Put/seqid=0 2024-12-03T02:20:17,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741900_1078 (size=5958) 2024-12-03T02:20:17,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741900_1078 (size=5958) 2024-12-03T02:20:17,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741900_1078 (size=5958) 2024-12-03T02:20:17,621 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/.tmp/b/0c3cef69438d4963a1ad3380eb0d36f0 2024-12-03T02:20:17,627 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/.tmp/b/0c3cef69438d4963a1ad3380eb0d36f0 as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/b/0c3cef69438d4963a1ad3380eb0d36f0 2024-12-03T02:20:17,633 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/b/0c3cef69438d4963a1ad3380eb0d36f0, entries=10, sequenceid=32, filesize=5.8 K 2024-12-03T02:20:17,633 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for a84ddab9aaac1e3bba10b1e54a42db42 in 30ms, sequenceid=32, compaction requested=false; wal=null 2024-12-03T02:20:17,634 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/recovered.edits/0000000000000000032 2024-12-03T02:20:17,635 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,635 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,636 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:17,637 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a84ddab9aaac1e3bba10b1e54a42db42 2024-12-03T02:20:17,639 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/a84ddab9aaac1e3bba10b1e54a42db42/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-03T02:20:17,640 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a84ddab9aaac1e3bba10b1e54a42db42; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62208634, jitterRate=-0.0730191171169281}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:17,641 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a84ddab9aaac1e3bba10b1e54a42db42: Writing region info on filesystem at 1733192417581Initializing all the Stores at 1733192417582 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192417582Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192417583 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192417583Obtaining lock to block concurrent updates at 1733192417603 (+20 ms)Preparing flush snapshotting stores in a84ddab9aaac1e3bba10b1e54a42db42 at 1733192417603Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733192417603Flushing stores of testReplayEditsWrittenViaHRegion,,1733192417287.a84ddab9aaac1e3bba10b1e54a42db42. at 1733192417603Flushing a84ddab9aaac1e3bba10b1e54a42db42/b: creating writer at 1733192417603Flushing a84ddab9aaac1e3bba10b1e54a42db42/b: appending metadata at 1733192417614 (+11 ms)Flushing a84ddab9aaac1e3bba10b1e54a42db42/b: closing flushed file at 1733192417615 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fd15c4c: reopening flushed file at 1733192417626 (+11 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for a84ddab9aaac1e3bba10b1e54a42db42 in 30ms, sequenceid=32, compaction requested=false; wal=null at 1733192417633 (+7 ms)Cleaning up temporary data from old regions at 1733192417635 (+2 ms)Region opened successfully at 1733192417640 (+5 ms) 2024-12-03T02:20:17,659 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=421 (was 411) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-537057715-172.17.0.3-1733192391930:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:38256 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:46036 [Waiting for operation #39] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:47734 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:45548 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-537057715-172.17.0.3-1733192391930:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:60958 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-537057715-172.17.0.3-1733192391930:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:57452 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1125 (was 1049) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=368 (was 368), ProcessCount=11 (was 11), AvailableMemoryMB=230 (was 238) 2024-12-03T02:20:17,659 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1125 is superior to 1024 2024-12-03T02:20:17,671 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=421, OpenFileDescriptor=1125, MaxFileDescriptor=1048576, SystemLoadAverage=368, ProcessCount=11, AvailableMemoryMB=230 2024-12-03T02:20:17,671 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1125 is superior to 1024 2024-12-03T02:20:17,685 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:17,687 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:17,688 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:20:17,690 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-24030810, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/hregion-24030810, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:17,702 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-24030810/hregion-24030810.1733192417691, exclude list is [], retry=0 2024-12-03T02:20:17,705 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:17,705 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:17,706 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:17,707 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-24030810/hregion-24030810.1733192417691 2024-12-03T02:20:17,710 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:39215:39215)] 2024-12-03T02:20:17,710 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c951778ec8be9a904bcc93486f884603, NAME => 'testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44855/hbase 2024-12-03T02:20:17,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741902_1080 (size=68) 2024-12-03T02:20:17,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741902_1080 (size=68) 2024-12-03T02:20:17,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741902_1080 (size=68) 2024-12-03T02:20:17,720 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:17,721 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,722 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c951778ec8be9a904bcc93486f884603 columnFamilyName a 2024-12-03T02:20:17,722 DEBUG [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:17,723 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(327): Store=c951778ec8be9a904bcc93486f884603/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:17,723 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,724 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c951778ec8be9a904bcc93486f884603 columnFamilyName b 2024-12-03T02:20:17,724 DEBUG [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:17,724 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(327): Store=c951778ec8be9a904bcc93486f884603/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:17,724 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,725 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c951778ec8be9a904bcc93486f884603 columnFamilyName c 2024-12-03T02:20:17,725 DEBUG [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:17,726 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(327): Store=c951778ec8be9a904bcc93486f884603/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:17,726 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,726 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,727 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,728 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,728 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,728 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:17,729 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,731 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:20:17,732 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c951778ec8be9a904bcc93486f884603; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63644429, jitterRate=-0.051624104380607605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:17,733 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c951778ec8be9a904bcc93486f884603: Writing region info on filesystem at 1733192417720Initializing all the Stores at 1733192417721 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192417721Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192417721Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192417721Cleaning up temporary data from old regions at 1733192417728 (+7 ms)Region opened successfully at 1733192417733 (+5 ms) 2024-12-03T02:20:17,733 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c951778ec8be9a904bcc93486f884603, disabling compactions & flushes 2024-12-03T02:20:17,733 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603. 2024-12-03T02:20:17,733 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603. 2024-12-03T02:20:17,733 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603. after waiting 0 ms 2024-12-03T02:20:17,733 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603. 2024-12-03T02:20:17,734 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603. 2024-12-03T02:20:17,734 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c951778ec8be9a904bcc93486f884603: Waiting for close lock at 1733192417733Disabling compacts and flushes for region at 1733192417733Disabling writes for close at 1733192417733Writing region close event to WAL at 1733192417733Closed at 1733192417733 2024-12-03T02:20:17,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741901_1079 (size=93) 2024-12-03T02:20:17,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741901_1079 (size=93) 2024-12-03T02:20:17,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741901_1079 (size=93) 2024-12-03T02:20:17,740 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T02:20:17,740 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-24030810:(num 1733192417691) 2024-12-03T02:20:17,740 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:17,742 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:17,757 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192417743, exclude list is [], retry=0 2024-12-03T02:20:17,760 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:17,760 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:17,761 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:17,762 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192417743 2024-12-03T02:20:17,763 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:36675:36675)] 2024-12-03T02:20:17,816 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c951778ec8be9a904bcc93486f884603, NAME => 'testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:17,819 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,819 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:17,819 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,819 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,821 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,822 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c951778ec8be9a904bcc93486f884603 columnFamilyName a 2024-12-03T02:20:17,822 DEBUG [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:17,822 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(327): Store=c951778ec8be9a904bcc93486f884603/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:17,822 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,823 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c951778ec8be9a904bcc93486f884603 columnFamilyName b 2024-12-03T02:20:17,823 DEBUG [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:17,824 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(327): Store=c951778ec8be9a904bcc93486f884603/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:17,824 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,825 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c951778ec8be9a904bcc93486f884603 columnFamilyName c 2024-12-03T02:20:17,825 DEBUG [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:17,825 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(327): Store=c951778ec8be9a904bcc93486f884603/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:17,826 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,826 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,827 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,828 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,829 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,829 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:17,830 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,831 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c951778ec8be9a904bcc93486f884603; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65978342, jitterRate=-0.01684609055519104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:17,831 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:17,831 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c951778ec8be9a904bcc93486f884603: Running coprocessor pre-open hook at 1733192417819Writing region info on filesystem at 1733192417819Initializing all the Stores at 1733192417820 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192417820Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192417820Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192417820Cleaning up temporary data from old regions at 1733192417829 (+9 ms)Running coprocessor post-open hooks at 1733192417831 (+2 ms)Region opened successfully at 1733192417831 2024-12-03T02:20:17,845 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c951778ec8be9a904bcc93486f884603 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-03T02:20:17,846 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c951778ec8be9a904bcc93486f884603/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:18,793 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T02:20:18,847 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c951778ec8be9a904bcc93486f884603/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:19,848 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c951778ec8be9a904bcc93486f884603/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:20,848 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c951778ec8be9a904bcc93486f884603/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:21,849 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c951778ec8be9a904bcc93486f884603/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:22,850 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c951778ec8be9a904bcc93486f884603/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:23,851 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c951778ec8be9a904bcc93486f884603/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:24,852 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c951778ec8be9a904bcc93486f884603/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:25,852 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c951778ec8be9a904bcc93486f884603/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:25,871 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T02:20:26,853 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c951778ec8be9a904bcc93486f884603/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:26,855 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c951778ec8be9a904bcc93486f884603: 2024-12-03T02:20:26,855 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603. 2024-12-03T02:20:26,874 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c951778ec8be9a904bcc93486f884603: 2024-12-03T02:20:26,874 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-03T02:20:26,874 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c951778ec8be9a904bcc93486f884603, disabling compactions & flushes 2024-12-03T02:20:26,874 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603. 2024-12-03T02:20:26,874 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603. 2024-12-03T02:20:26,874 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603. after waiting 0 ms 2024-12-03T02:20:26,874 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603. 2024-12-03T02:20:26,875 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603. 2024-12-03T02:20:26,875 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603. 2024-12-03T02:20:26,875 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c951778ec8be9a904bcc93486f884603: Waiting for close lock at 1733192426874Running coprocessor pre-close hooks at 1733192426874Disabling compacts and flushes for region at 1733192426874Disabling writes for close at 1733192426874Writing region close event to WAL at 1733192426874Running coprocessor post-close hooks at 1733192426875 (+1 ms)Closed at 1733192426875 2024-12-03T02:20:26,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741903_1081 (size=3941) 2024-12-03T02:20:26,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741903_1081 (size=3941) 2024-12-03T02:20:26,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741903_1081 (size=3941) 2024-12-03T02:20:26,893 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44855/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192417743, size=3.8 K (3941bytes) 2024-12-03T02:20:26,893 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44855/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192417743 2024-12-03T02:20:26,894 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44855/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192417743 after 1ms 2024-12-03T02:20:26,897 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44855/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192417743 took 4ms 2024-12-03T02:20:26,899 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44855/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192417743 so closing down 2024-12-03T02:20:26,899 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T02:20:26,900 INFO [Time-limited test-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733192417743.temp 2024-12-03T02:20:26,901 INFO [Time-limited test-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/recovered.edits/0000000000000000004-wal.1733192417743.temp 2024-12-03T02:20:26,902 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T02:20:26,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741904_1082 (size=2983) 2024-12-03T02:20:26,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741904_1082 (size=2983) 2024-12-03T02:20:26,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741904_1082 (size=2983) 2024-12-03T02:20:26,921 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/recovered.edits/0000000000000000004-wal.1733192417743.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-03T02:20:26,922 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/recovered.edits/0000000000000000004-wal.1733192417743.temp to hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/recovered.edits/0000000000000000026 2024-12-03T02:20:26,922 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 25 ms; skipped=3; WAL=hdfs://localhost:44855/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192417743, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-12-03T02:20:26,922 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44855/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192417743, journal: Splitting hdfs://localhost:44855/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192417743, size=3.8 K (3941bytes) at 1733192426893Finishing writing output for hdfs://localhost:44855/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192417743 so closing down at 1733192426899 (+6 ms)Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/recovered.edits/0000000000000000004-wal.1733192417743.temp at 1733192426901 (+2 ms)3 split writer threads finished at 1733192426902 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/recovered.edits/0000000000000000004-wal.1733192417743.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733192426921 (+19 ms)Rename recovered edits hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/recovered.edits/0000000000000000004-wal.1733192417743.temp to hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/recovered.edits/0000000000000000026 at 1733192426922 (+1 ms)Processed 23 edits across 1 Regions in 25 ms; skipped=3; WAL=hdfs://localhost:44855/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192417743, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1733192426922 2024-12-03T02:20:26,924 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44855/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192417743 to hdfs://localhost:44855/hbase/oldWALs/wal.1733192417743 2024-12-03T02:20:26,925 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/recovered.edits/0000000000000000026 2024-12-03T02:20:26,925 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:26,927 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:26,938 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192426927, exclude list is [], retry=0 2024-12-03T02:20:26,941 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:26,941 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:26,941 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:26,943 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192426927 2024-12-03T02:20:26,943 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:39215:39215)] 2024-12-03T02:20:26,943 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c951778ec8be9a904bcc93486f884603, NAME => 'testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:26,944 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:26,944 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:26,944 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:26,944 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:26,946 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:26,947 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c951778ec8be9a904bcc93486f884603 columnFamilyName a 2024-12-03T02:20:26,947 DEBUG [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:26,947 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(327): Store=c951778ec8be9a904bcc93486f884603/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:26,947 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:26,948 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c951778ec8be9a904bcc93486f884603 columnFamilyName b 2024-12-03T02:20:26,948 DEBUG [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:26,949 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(327): Store=c951778ec8be9a904bcc93486f884603/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:26,949 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:26,950 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c951778ec8be9a904bcc93486f884603 columnFamilyName c 2024-12-03T02:20:26,950 DEBUG [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:26,951 INFO [StoreOpener-c951778ec8be9a904bcc93486f884603-1 {}] regionserver.HStore(327): Store=c951778ec8be9a904bcc93486f884603/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:26,951 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:26,952 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:26,954 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:26,954 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/recovered.edits/0000000000000000026 2024-12-03T02:20:26,958 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/recovered.edits/0000000000000000026 2024-12-03T02:20:26,958 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c951778ec8be9a904bcc93486f884603 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-03T02:20:26,975 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/.tmp/a/37bee905bb7c4ae98d7290da9a5e36d3 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733192426863/Put/seqid=0 2024-12-03T02:20:26,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741906_1084 (size=5523) 2024-12-03T02:20:26,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741906_1084 (size=5523) 2024-12-03T02:20:26,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741906_1084 (size=5523) 2024-12-03T02:20:27,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-03T02:20:27,042 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-03T02:20:27,387 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/.tmp/a/37bee905bb7c4ae98d7290da9a5e36d3 2024-12-03T02:20:27,410 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/.tmp/b/32d22ff398a0421dbdc81b39f0eb5648 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733192426856/Put/seqid=0 2024-12-03T02:20:27,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741907_1085 (size=5524) 2024-12-03T02:20:27,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741907_1085 (size=5524) 2024-12-03T02:20:27,425 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/.tmp/b/32d22ff398a0421dbdc81b39f0eb5648 2024-12-03T02:20:27,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741907_1085 (size=5524) 2024-12-03T02:20:27,451 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/.tmp/c/9a019f0ef1d34dfab42a1e2cf57219a2 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733192426860/Put/seqid=0 2024-12-03T02:20:27,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741908_1086 (size=5457) 2024-12-03T02:20:27,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741908_1086 (size=5457) 2024-12-03T02:20:27,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741908_1086 (size=5457) 2024-12-03T02:20:27,473 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/.tmp/c/9a019f0ef1d34dfab42a1e2cf57219a2 2024-12-03T02:20:27,484 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/.tmp/a/37bee905bb7c4ae98d7290da9a5e36d3 as hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/a/37bee905bb7c4ae98d7290da9a5e36d3 2024-12-03T02:20:27,491 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/a/37bee905bb7c4ae98d7290da9a5e36d3, entries=7, sequenceid=26, filesize=5.4 K 2024-12-03T02:20:27,492 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/.tmp/b/32d22ff398a0421dbdc81b39f0eb5648 as hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/b/32d22ff398a0421dbdc81b39f0eb5648 2024-12-03T02:20:27,498 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/b/32d22ff398a0421dbdc81b39f0eb5648, entries=7, sequenceid=26, filesize=5.4 K 2024-12-03T02:20:27,499 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/.tmp/c/9a019f0ef1d34dfab42a1e2cf57219a2 as hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/c/9a019f0ef1d34dfab42a1e2cf57219a2 2024-12-03T02:20:27,504 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/c/9a019f0ef1d34dfab42a1e2cf57219a2, entries=6, sequenceid=26, filesize=5.3 K 2024-12-03T02:20:27,505 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for c951778ec8be9a904bcc93486f884603 in 547ms, sequenceid=26, compaction requested=false; wal=null 2024-12-03T02:20:27,505 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/recovered.edits/0000000000000000026 2024-12-03T02:20:27,506 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:27,506 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:27,507 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:27,508 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:27,511 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testReplayEditsAfterAbortingFlush/c951778ec8be9a904bcc93486f884603/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-03T02:20:27,512 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c951778ec8be9a904bcc93486f884603; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68778166, jitterRate=0.02487453818321228}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:27,512 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c951778ec8be9a904bcc93486f884603 2024-12-03T02:20:27,512 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c951778ec8be9a904bcc93486f884603: Running coprocessor pre-open hook at 1733192426944Writing region info on filesystem at 1733192426944Initializing all the Stores at 1733192426945 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192426945Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192426945Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192426945Obtaining lock to block concurrent updates at 1733192426958 (+13 ms)Preparing flush snapshotting stores in c951778ec8be9a904bcc93486f884603 at 1733192426958Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733192426959 (+1 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1733192417686.c951778ec8be9a904bcc93486f884603. at 1733192426959Flushing c951778ec8be9a904bcc93486f884603/a: creating writer at 1733192426959Flushing c951778ec8be9a904bcc93486f884603/a: appending metadata at 1733192426975 (+16 ms)Flushing c951778ec8be9a904bcc93486f884603/a: closing flushed file at 1733192426975Flushing c951778ec8be9a904bcc93486f884603/b: creating writer at 1733192427394 (+419 ms)Flushing c951778ec8be9a904bcc93486f884603/b: appending metadata at 1733192427409 (+15 ms)Flushing c951778ec8be9a904bcc93486f884603/b: closing flushed file at 1733192427409Flushing c951778ec8be9a904bcc93486f884603/c: creating writer at 1733192427433 (+24 ms)Flushing c951778ec8be9a904bcc93486f884603/c: appending metadata at 1733192427450 (+17 ms)Flushing c951778ec8be9a904bcc93486f884603/c: closing flushed file at 1733192427451 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22eb3740: reopening flushed file at 1733192427482 (+31 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c147b: reopening flushed file at 1733192427491 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fe3ce6: reopening flushed file at 1733192427498 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for c951778ec8be9a904bcc93486f884603 in 547ms, sequenceid=26, compaction requested=false; wal=null at 1733192427505 (+7 ms)Cleaning up temporary data from old regions at 1733192427506 (+1 ms)Running coprocessor post-open hooks at 1733192427512 (+6 ms)Region opened successfully at 1733192427512 2024-12-03T02:20:27,541 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=418 (was 421), OpenFileDescriptor=1180 (was 1125) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=425 (was 368) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=232 (was 230) - AvailableMemoryMB LEAK? - 2024-12-03T02:20:27,541 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-12-03T02:20:27,556 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=418, OpenFileDescriptor=1180, MaxFileDescriptor=1048576, SystemLoadAverage=425, ProcessCount=11, AvailableMemoryMB=230 2024-12-03T02:20:27,556 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-12-03T02:20:27,576 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:27,578 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:27,579 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:20:27,582 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23842102, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/hregion-23842102, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:27,593 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23842102/hregion-23842102.1733192427582, exclude list is [], retry=0 2024-12-03T02:20:27,596 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:27,596 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:27,596 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:27,599 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23842102/hregion-23842102.1733192427582 2024-12-03T02:20:27,599 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:36675:36675)] 2024-12-03T02:20:27,599 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => dbbba4642e13b4e1aa09038db9fc4d89, NAME => 'testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44855/hbase 2024-12-03T02:20:27,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741910_1088 (size=61) 2024-12-03T02:20:27,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741910_1088 (size=61) 2024-12-03T02:20:27,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741910_1088 (size=61) 2024-12-03T02:20:27,611 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:27,612 INFO [StoreOpener-dbbba4642e13b4e1aa09038db9fc4d89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:27,613 INFO [StoreOpener-dbbba4642e13b4e1aa09038db9fc4d89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbbba4642e13b4e1aa09038db9fc4d89 columnFamilyName a 2024-12-03T02:20:27,613 DEBUG [StoreOpener-dbbba4642e13b4e1aa09038db9fc4d89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:27,614 INFO [StoreOpener-dbbba4642e13b4e1aa09038db9fc4d89-1 {}] regionserver.HStore(327): Store=dbbba4642e13b4e1aa09038db9fc4d89/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:27,614 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:27,614 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:27,615 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:27,615 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:27,615 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:27,617 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:27,620 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:20:27,620 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dbbba4642e13b4e1aa09038db9fc4d89; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65105652, jitterRate=-0.02985018491744995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T02:20:27,621 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dbbba4642e13b4e1aa09038db9fc4d89: Writing region info on filesystem at 1733192427611Initializing all the Stores at 1733192427612 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192427612Cleaning up temporary data from old regions at 1733192427615 (+3 ms)Region opened successfully at 1733192427621 (+6 ms) 2024-12-03T02:20:27,621 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dbbba4642e13b4e1aa09038db9fc4d89, disabling compactions & flushes 2024-12-03T02:20:27,621 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89. 2024-12-03T02:20:27,621 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89. 2024-12-03T02:20:27,621 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89. after waiting 0 ms 2024-12-03T02:20:27,621 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89. 2024-12-03T02:20:27,622 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89. 2024-12-03T02:20:27,622 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dbbba4642e13b4e1aa09038db9fc4d89: Waiting for close lock at 1733192427621Disabling compacts and flushes for region at 1733192427621Disabling writes for close at 1733192427621Writing region close event to WAL at 1733192427621Closed at 1733192427621 2024-12-03T02:20:27,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741909_1087 (size=93) 2024-12-03T02:20:27,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741909_1087 (size=93) 2024-12-03T02:20:27,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741909_1087 (size=93) 2024-12-03T02:20:27,627 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T02:20:27,627 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23842102:(num 1733192427582) 2024-12-03T02:20:27,627 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:27,629 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:27,642 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192427629, exclude list is [], retry=0 2024-12-03T02:20:27,645 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:27,645 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:27,646 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:27,647 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192427629 2024-12-03T02:20:27,647 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:43847:43847)] 2024-12-03T02:20:27,647 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dbbba4642e13b4e1aa09038db9fc4d89, NAME => 'testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:27,648 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:27,648 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:27,648 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:27,649 INFO [StoreOpener-dbbba4642e13b4e1aa09038db9fc4d89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:27,650 INFO [StoreOpener-dbbba4642e13b4e1aa09038db9fc4d89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbbba4642e13b4e1aa09038db9fc4d89 columnFamilyName a 2024-12-03T02:20:27,651 DEBUG [StoreOpener-dbbba4642e13b4e1aa09038db9fc4d89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:27,651 INFO [StoreOpener-dbbba4642e13b4e1aa09038db9fc4d89-1 {}] regionserver.HStore(327): Store=dbbba4642e13b4e1aa09038db9fc4d89/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:27,651 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:27,652 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:27,654 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:27,654 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:27,654 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:27,656 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:27,657 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dbbba4642e13b4e1aa09038db9fc4d89; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65834782, jitterRate=-0.01898530125617981}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T02:20:27,658 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dbbba4642e13b4e1aa09038db9fc4d89: Writing region info on filesystem at 1733192427648Initializing all the Stores at 1733192427649 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192427649Cleaning up temporary data from old regions at 1733192427654 (+5 ms)Region opened successfully at 1733192427658 (+4 ms) 2024-12-03T02:20:27,669 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dbbba4642e13b4e1aa09038db9fc4d89, disabling compactions & flushes 2024-12-03T02:20:27,669 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89. 2024-12-03T02:20:27,669 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89. 2024-12-03T02:20:27,669 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89. after waiting 0 ms 2024-12-03T02:20:27,669 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89. 2024-12-03T02:20:27,670 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89. 2024-12-03T02:20:27,670 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89. 2024-12-03T02:20:27,670 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dbbba4642e13b4e1aa09038db9fc4d89: Waiting for close lock at 1733192427669Disabling compacts and flushes for region at 1733192427669Disabling writes for close at 1733192427669Writing region close event to WAL at 1733192427670 (+1 ms)Closed at 1733192427670 2024-12-03T02:20:27,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741911_1089 (size=1623) 2024-12-03T02:20:27,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741911_1089 (size=1623) 2024-12-03T02:20:27,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741911_1089 (size=1623) 2024-12-03T02:20:27,694 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44855/hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192427629, size=1.6 K (1623bytes) 2024-12-03T02:20:27,694 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44855/hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192427629 2024-12-03T02:20:27,695 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44855/hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192427629 after 1ms 2024-12-03T02:20:27,697 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44855/hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192427629 took 3ms 2024-12-03T02:20:27,699 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44855/hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192427629 so closing down 2024-12-03T02:20:27,699 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T02:20:27,700 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733192427629.temp 2024-12-03T02:20:27,701 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/recovered.edits/0000000000000000003-wal.1733192427629.temp 2024-12-03T02:20:27,701 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T02:20:27,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741912_1090 (size=1623) 2024-12-03T02:20:27,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741912_1090 (size=1623) 2024-12-03T02:20:27,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741912_1090 (size=1623) 2024-12-03T02:20:27,708 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/recovered.edits/0000000000000000003-wal.1733192427629.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-03T02:20:27,709 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/recovered.edits/0000000000000000003-wal.1733192427629.temp to hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/recovered.edits/0000000000000000012 2024-12-03T02:20:27,709 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192427629, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-12-03T02:20:27,709 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44855/hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192427629, journal: Splitting hdfs://localhost:44855/hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192427629, size=1.6 K (1623bytes) at 1733192427694Finishing writing output for hdfs://localhost:44855/hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192427629 so closing down at 1733192427699 (+5 ms)Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/recovered.edits/0000000000000000003-wal.1733192427629.temp at 1733192427701 (+2 ms)3 split writer threads finished at 1733192427701Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/recovered.edits/0000000000000000003-wal.1733192427629.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733192427708 (+7 ms)Rename recovered edits hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/recovered.edits/0000000000000000003-wal.1733192427629.temp to hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/recovered.edits/0000000000000000012 at 1733192427709 (+1 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192427629, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1733192427709 2024-12-03T02:20:27,711 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44855/hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192427629 to hdfs://localhost:44855/hbase/oldWALs/wal.1733192427629 2024-12-03T02:20:27,712 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/recovered.edits/0000000000000000012 2024-12-03T02:20:28,089 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:28,091 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:28,104 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192428092, exclude list is [], retry=0 2024-12-03T02:20:28,106 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:28,107 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:28,107 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:28,109 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192428092 2024-12-03T02:20:28,109 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:43847:43847)] 2024-12-03T02:20:28,109 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dbbba4642e13b4e1aa09038db9fc4d89, NAME => 'testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:28,109 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:28,110 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,110 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,113 INFO [StoreOpener-dbbba4642e13b4e1aa09038db9fc4d89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,114 INFO [StoreOpener-dbbba4642e13b4e1aa09038db9fc4d89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbbba4642e13b4e1aa09038db9fc4d89 columnFamilyName a 2024-12-03T02:20:28,114 DEBUG [StoreOpener-dbbba4642e13b4e1aa09038db9fc4d89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:28,115 INFO [StoreOpener-dbbba4642e13b4e1aa09038db9fc4d89-1 {}] regionserver.HStore(327): Store=dbbba4642e13b4e1aa09038db9fc4d89/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:28,115 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,116 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,117 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,118 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/recovered.edits/0000000000000000012 2024-12-03T02:20:28,121 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/recovered.edits/0000000000000000012 2024-12-03T02:20:28,121 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dbbba4642e13b4e1aa09038db9fc4d89 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-03T02:20:28,138 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/.tmp/a/9555cb24676c4a9481cb1aec5297a695 is 79, key is testDatalossWhenInputError/a:x0/1733192427658/Put/seqid=0 2024-12-03T02:20:28,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741914_1092 (size=5808) 2024-12-03T02:20:28,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741914_1092 (size=5808) 2024-12-03T02:20:28,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741914_1092 (size=5808) 2024-12-03T02:20:28,146 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/.tmp/a/9555cb24676c4a9481cb1aec5297a695 2024-12-03T02:20:28,157 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/.tmp/a/9555cb24676c4a9481cb1aec5297a695 as hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/a/9555cb24676c4a9481cb1aec5297a695 2024-12-03T02:20:28,166 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/a/9555cb24676c4a9481cb1aec5297a695, entries=10, sequenceid=12, filesize=5.7 K 2024-12-03T02:20:28,167 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for dbbba4642e13b4e1aa09038db9fc4d89 in 46ms, sequenceid=12, compaction requested=false; wal=null 2024-12-03T02:20:28,168 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/recovered.edits/0000000000000000012 2024-12-03T02:20:28,169 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,169 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,172 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,174 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-03T02:20:28,175 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dbbba4642e13b4e1aa09038db9fc4d89; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64997792, jitterRate=-0.03145742416381836}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T02:20:28,175 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dbbba4642e13b4e1aa09038db9fc4d89: Writing region info on filesystem at 1733192428110Initializing all the Stores at 1733192428112 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192428112Obtaining lock to block concurrent updates at 1733192428121 (+9 ms)Preparing flush snapshotting stores in dbbba4642e13b4e1aa09038db9fc4d89 at 1733192428121Finished memstore snapshotting testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733192428122 (+1 ms)Flushing stores of testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89. at 1733192428122Flushing dbbba4642e13b4e1aa09038db9fc4d89/a: creating writer at 1733192428122Flushing dbbba4642e13b4e1aa09038db9fc4d89/a: appending metadata at 1733192428138 (+16 ms)Flushing dbbba4642e13b4e1aa09038db9fc4d89/a: closing flushed file at 1733192428138Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@778693f: reopening flushed file at 1733192428156 (+18 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for dbbba4642e13b4e1aa09038db9fc4d89 in 46ms, sequenceid=12, compaction requested=false; wal=null at 1733192428167 (+11 ms)Cleaning up temporary data from old regions at 1733192428169 (+2 ms)Region opened successfully at 1733192428175 (+6 ms) 2024-12-03T02:20:28,178 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dbbba4642e13b4e1aa09038db9fc4d89, NAME => 'testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:28,178 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733192427577.dbbba4642e13b4e1aa09038db9fc4d89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:28,178 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,178 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,179 INFO [StoreOpener-dbbba4642e13b4e1aa09038db9fc4d89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,180 INFO [StoreOpener-dbbba4642e13b4e1aa09038db9fc4d89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbbba4642e13b4e1aa09038db9fc4d89 columnFamilyName a 2024-12-03T02:20:28,180 DEBUG [StoreOpener-dbbba4642e13b4e1aa09038db9fc4d89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:28,185 DEBUG [StoreOpener-dbbba4642e13b4e1aa09038db9fc4d89-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/a/9555cb24676c4a9481cb1aec5297a695 2024-12-03T02:20:28,185 INFO [StoreOpener-dbbba4642e13b4e1aa09038db9fc4d89-1 {}] regionserver.HStore(327): Store=dbbba4642e13b4e1aa09038db9fc4d89/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:28,185 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,186 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,187 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,188 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,188 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,190 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dbbba4642e13b4e1aa09038db9fc4d89 2024-12-03T02:20:28,193 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testDatalossWhenInputError/dbbba4642e13b4e1aa09038db9fc4d89/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-03T02:20:28,194 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dbbba4642e13b4e1aa09038db9fc4d89; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59226277, jitterRate=-0.11745969951152802}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T02:20:28,194 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dbbba4642e13b4e1aa09038db9fc4d89: Writing region info on filesystem at 1733192428178Initializing all the Stores at 1733192428179 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192428179Cleaning up temporary data from old regions at 1733192428188 (+9 ms)Region opened successfully at 1733192428194 (+6 ms) 2024-12-03T02:20:28,213 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=428 (was 418) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-537057715-172.17.0.3-1733192391930:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:45554 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:45494 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:32902 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:59156 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-537057715-172.17.0.3-1733192391930:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-537057715-172.17.0.3-1733192391930:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:32850 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1260 (was 1180) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=425 (was 425), ProcessCount=11 (was 11), AvailableMemoryMB=264 (was 230) - AvailableMemoryMB LEAK? - 2024-12-03T02:20:28,213 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1260 is superior to 1024 2024-12-03T02:20:28,228 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=428, OpenFileDescriptor=1260, MaxFileDescriptor=1048576, SystemLoadAverage=425, ProcessCount=11, AvailableMemoryMB=263 2024-12-03T02:20:28,229 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1260 is superior to 1024 2024-12-03T02:20:28,246 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:28,248 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:28,249 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:20:28,253 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-78304549, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/hregion-78304549, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:28,272 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-78304549/hregion-78304549.1733192428253, exclude list is [], retry=0 2024-12-03T02:20:28,275 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:28,275 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:28,284 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:28,290 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-78304549/hregion-78304549.1733192428253 2024-12-03T02:20:28,291 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:36675:36675)] 2024-12-03T02:20:28,291 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2d7ca1d81f9ffdce3fe685e573257560, NAME => 'testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44855/hbase 2024-12-03T02:20:28,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741916_1094 (size=63) 2024-12-03T02:20:28,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741916_1094 (size=63) 2024-12-03T02:20:28,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741916_1094 (size=63) 2024-12-03T02:20:28,311 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:28,315 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,318 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d7ca1d81f9ffdce3fe685e573257560 columnFamilyName a 2024-12-03T02:20:28,318 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:28,318 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(327): Store=2d7ca1d81f9ffdce3fe685e573257560/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:28,319 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,320 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d7ca1d81f9ffdce3fe685e573257560 columnFamilyName b 2024-12-03T02:20:28,320 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:28,321 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(327): Store=2d7ca1d81f9ffdce3fe685e573257560/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:28,321 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,322 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d7ca1d81f9ffdce3fe685e573257560 columnFamilyName c 2024-12-03T02:20:28,323 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:28,323 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(327): Store=2d7ca1d81f9ffdce3fe685e573257560/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:28,323 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,324 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,324 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,326 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,326 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,326 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:28,328 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,336 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:20:28,337 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2d7ca1d81f9ffdce3fe685e573257560; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64169888, jitterRate=-0.04379415512084961}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:28,337 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2d7ca1d81f9ffdce3fe685e573257560: Writing region info on filesystem at 1733192428311Initializing all the Stores at 1733192428313 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192428313Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192428315 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192428315Cleaning up temporary data from old regions at 1733192428326 (+11 ms)Region opened successfully at 1733192428337 (+11 ms) 2024-12-03T02:20:28,338 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2d7ca1d81f9ffdce3fe685e573257560, disabling compactions & flushes 2024-12-03T02:20:28,338 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560. 2024-12-03T02:20:28,338 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560. 2024-12-03T02:20:28,338 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560. after waiting 0 ms 2024-12-03T02:20:28,338 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560. 2024-12-03T02:20:28,338 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560. 2024-12-03T02:20:28,338 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2d7ca1d81f9ffdce3fe685e573257560: Waiting for close lock at 1733192428338Disabling compacts and flushes for region at 1733192428338Disabling writes for close at 1733192428338Writing region close event to WAL at 1733192428338Closed at 1733192428338 2024-12-03T02:20:28,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741915_1093 (size=93) 2024-12-03T02:20:28,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741915_1093 (size=93) 2024-12-03T02:20:28,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741915_1093 (size=93) 2024-12-03T02:20:28,351 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T02:20:28,351 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-78304549:(num 1733192428253) 2024-12-03T02:20:28,351 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:28,354 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:28,367 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354, exclude list is [], retry=0 2024-12-03T02:20:28,370 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:28,370 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:28,371 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:28,372 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354 2024-12-03T02:20:28,373 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:43847:43847)] 2024-12-03T02:20:28,373 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d7ca1d81f9ffdce3fe685e573257560, NAME => 'testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:28,373 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:28,373 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,373 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,374 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,375 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d7ca1d81f9ffdce3fe685e573257560 columnFamilyName a 2024-12-03T02:20:28,375 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:28,376 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(327): Store=2d7ca1d81f9ffdce3fe685e573257560/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:28,376 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,377 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d7ca1d81f9ffdce3fe685e573257560 columnFamilyName b 2024-12-03T02:20:28,377 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:28,377 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(327): Store=2d7ca1d81f9ffdce3fe685e573257560/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:28,377 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,378 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d7ca1d81f9ffdce3fe685e573257560 columnFamilyName c 2024-12-03T02:20:28,378 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:28,378 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(327): Store=2d7ca1d81f9ffdce3fe685e573257560/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:28,378 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,379 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,381 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,383 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,383 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,383 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:28,384 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:28,386 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2d7ca1d81f9ffdce3fe685e573257560; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73383324, jitterRate=0.09349673986434937}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:28,386 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2d7ca1d81f9ffdce3fe685e573257560: Writing region info on filesystem at 1733192428373Initializing all the Stores at 1733192428374 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192428374Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192428374Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192428374Cleaning up temporary data from old regions at 1733192428383 (+9 ms)Region opened successfully at 1733192428386 (+3 ms) 2024-12-03T02:20:28,390 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733192428390/Put/seqid=0 2024-12-03T02:20:28,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741918_1096 (size=4875) 2024-12-03T02:20:28,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741918_1096 (size=4875) 2024-12-03T02:20:28,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741918_1096 (size=4875) 2024-12-03T02:20:28,400 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733192428400/Put/seqid=0 2024-12-03T02:20:28,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741919_1097 (size=4875) 2024-12-03T02:20:28,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741919_1097 (size=4875) 2024-12-03T02:20:28,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741919_1097 (size=4875) 2024-12-03T02:20:28,408 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733192428408/Put/seqid=0 2024-12-03T02:20:28,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741920_1098 (size=4875) 2024-12-03T02:20:28,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741920_1098 (size=4875) 2024-12-03T02:20:28,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741920_1098 (size=4875) 2024-12-03T02:20:28,419 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44855/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 2d7ca1d81f9ffdce3fe685e573257560/a 2024-12-03T02:20:28,423 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-03T02:20:28,423 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T02:20:28,423 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44855/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 2d7ca1d81f9ffdce3fe685e573257560/a 2024-12-03T02:20:28,427 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-03T02:20:28,427 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T02:20:28,427 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44855/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 2d7ca1d81f9ffdce3fe685e573257560/a 2024-12-03T02:20:28,431 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-03T02:20:28,432 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T02:20:28,432 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2d7ca1d81f9ffdce3fe685e573257560 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-03T02:20:28,455 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/.tmp/a/194ff877782f4b66b370cd57463b3d21 is 55, key is testCompactedBulkLoadedFiles/a:a/1733192428386/Put/seqid=0 2024-12-03T02:20:28,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741921_1099 (size=5107) 2024-12-03T02:20:28,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741921_1099 (size=5107) 2024-12-03T02:20:28,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741921_1099 (size=5107) 2024-12-03T02:20:28,481 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/.tmp/a/194ff877782f4b66b370cd57463b3d21 2024-12-03T02:20:28,490 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/.tmp/a/194ff877782f4b66b370cd57463b3d21 as hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/194ff877782f4b66b370cd57463b3d21 2024-12-03T02:20:28,503 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/194ff877782f4b66b370cd57463b3d21, entries=1, sequenceid=4, filesize=5.0 K 2024-12-03T02:20:28,512 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 2d7ca1d81f9ffdce3fe685e573257560 in 79ms, sequenceid=4, compaction requested=false 2024-12-03T02:20:28,512 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2d7ca1d81f9ffdce3fe685e573257560: 2024-12-03T02:20:28,514 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1ef26df5f5f044b9a477c72197e8e89b_SeqId_4_ 2024-12-03T02:20:28,517 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/f4ea0a5ffcdb4d76819b5870561d027a_SeqId_4_ 2024-12-03T02:20:28,524 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1bd7759c8aa24cc89d3c35d8708fa8cc_SeqId_4_ 2024-12-03T02:20:28,525 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44855/hbase/testCompactedBulkLoadedFiles/hfile0 into 2d7ca1d81f9ffdce3fe685e573257560/a as hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1ef26df5f5f044b9a477c72197e8e89b_SeqId_4_ - updating store file list. 2024-12-03T02:20:28,535 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1ef26df5f5f044b9a477c72197e8e89b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T02:20:28,535 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1ef26df5f5f044b9a477c72197e8e89b_SeqId_4_ into 2d7ca1d81f9ffdce3fe685e573257560/a 2024-12-03T02:20:28,535 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44855/hbase/testCompactedBulkLoadedFiles/hfile0 into 2d7ca1d81f9ffdce3fe685e573257560/a (new location: hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1ef26df5f5f044b9a477c72197e8e89b_SeqId_4_) 2024-12-03T02:20:28,539 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44855/hbase/testCompactedBulkLoadedFiles/hfile1 into 2d7ca1d81f9ffdce3fe685e573257560/a as hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/f4ea0a5ffcdb4d76819b5870561d027a_SeqId_4_ - updating store file list. 2024-12-03T02:20:28,545 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for f4ea0a5ffcdb4d76819b5870561d027a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T02:20:28,545 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/f4ea0a5ffcdb4d76819b5870561d027a_SeqId_4_ into 2d7ca1d81f9ffdce3fe685e573257560/a 2024-12-03T02:20:28,545 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44855/hbase/testCompactedBulkLoadedFiles/hfile1 into 2d7ca1d81f9ffdce3fe685e573257560/a (new location: hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/f4ea0a5ffcdb4d76819b5870561d027a_SeqId_4_) 2024-12-03T02:20:28,547 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44855/hbase/testCompactedBulkLoadedFiles/hfile2 into 2d7ca1d81f9ffdce3fe685e573257560/a as hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1bd7759c8aa24cc89d3c35d8708fa8cc_SeqId_4_ - updating store file list. 2024-12-03T02:20:28,554 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1bd7759c8aa24cc89d3c35d8708fa8cc_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T02:20:28,555 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1bd7759c8aa24cc89d3c35d8708fa8cc_SeqId_4_ into 2d7ca1d81f9ffdce3fe685e573257560/a 2024-12-03T02:20:28,555 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44855/hbase/testCompactedBulkLoadedFiles/hfile2 into 2d7ca1d81f9ffdce3fe685e573257560/a (new location: hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1bd7759c8aa24cc89d3c35d8708fa8cc_SeqId_4_) 2024-12-03T02:20:28,593 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:20:28,594 DEBUG [Time-limited test {}] regionserver.HStore(1541): 2d7ca1d81f9ffdce3fe685e573257560/a is initiating major compaction (all files) 2024-12-03T02:20:28,594 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 2d7ca1d81f9ffdce3fe685e573257560/a in testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560. 2024-12-03T02:20:28,594 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/194ff877782f4b66b370cd57463b3d21, hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1ef26df5f5f044b9a477c72197e8e89b_SeqId_4_, hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/f4ea0a5ffcdb4d76819b5870561d027a_SeqId_4_, hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1bd7759c8aa24cc89d3c35d8708fa8cc_SeqId_4_] into tmpdir=hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/.tmp, totalSize=19.3 K 2024-12-03T02:20:28,599 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 194ff877782f4b66b370cd57463b3d21, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733192428386 2024-12-03T02:20:28,603 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1ef26df5f5f044b9a477c72197e8e89b_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T02:20:28,607 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f4ea0a5ffcdb4d76819b5870561d027a_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T02:20:28,612 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1bd7759c8aa24cc89d3c35d8708fa8cc_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T02:20:28,640 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/.tmp/a/d7f20d27229b42b09d0c099ff899847c is 55, key is testCompactedBulkLoadedFiles/a:a/1733192428386/Put/seqid=0 2024-12-03T02:20:28,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741922_1100 (size=6154) 2024-12-03T02:20:28,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741922_1100 (size=6154) 2024-12-03T02:20:28,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741922_1100 (size=6154) 2024-12-03T02:20:28,715 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/.tmp/a/d7f20d27229b42b09d0c099ff899847c as hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/d7f20d27229b42b09d0c099ff899847c 2024-12-03T02:20:28,737 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 2d7ca1d81f9ffdce3fe685e573257560/a of 2d7ca1d81f9ffdce3fe685e573257560 into d7f20d27229b42b09d0c099ff899847c(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:20:28,737 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 2d7ca1d81f9ffdce3fe685e573257560: 2024-12-03T02:20:28,737 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-03T02:20:28,737 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-03T02:20:28,788 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44855/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354, size=0 (0bytes) 2024-12-03T02:20:28,788 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44855/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354 might be still open, length is 0 2024-12-03T02:20:28,788 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44855/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354 2024-12-03T02:20:28,790 WARN [IPC Server handler 0 on default port 44855 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-03T02:20:28,790 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44855/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354 after 2ms 2024-12-03T02:20:31,767 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:59188 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36885:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59188 dst: /127.0.0.1:36885 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36885 remote=/127.0.0.1:59188]. Total timeout mills is 60000, 56966 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:31,768 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:32940 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:35125:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:32940 dst: /127.0.0.1:35125 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:31,768 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:45586 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:43037:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45586 dst: /127.0.0.1:43037 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:31,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741917_1101 (size=1659) 2024-12-03T02:20:31,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741917_1101 (size=1659) 2024-12-03T02:20:31,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741917_1101 (size=1659) 2024-12-03T02:20:32,398 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T02:20:32,792 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44855/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354 after 4004ms 2024-12-03T02:20:32,799 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44855/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354 took 4011ms 2024-12-03T02:20:32,801 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44855/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354; continuing. 2024-12-03T02:20:32,802 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44855/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354 so closing down 2024-12-03T02:20:32,802 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T02:20:32,803 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733192428354.temp 2024-12-03T02:20:32,805 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/recovered.edits/0000000000000000003-wal.1733192428354.temp 2024-12-03T02:20:32,805 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T02:20:32,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741923_1102 (size=698) 2024-12-03T02:20:32,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741923_1102 (size=698) 2024-12-03T02:20:32,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741923_1102 (size=698) 2024-12-03T02:20:32,813 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/recovered.edits/0000000000000000003-wal.1733192428354.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T02:20:32,815 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/recovered.edits/0000000000000000003-wal.1733192428354.temp to hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/recovered.edits/0000000000000000008 2024-12-03T02:20:32,816 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:44855/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T02:20:32,816 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44855/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354, journal: Splitting hdfs://localhost:44855/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354, size=0 (0bytes) at 1733192428788Finishing writing output for hdfs://localhost:44855/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354 so closing down at 1733192432802 (+4014 ms)Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/recovered.edits/0000000000000000003-wal.1733192428354.temp at 1733192432805 (+3 ms)3 split writer threads finished at 1733192432805Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/recovered.edits/0000000000000000003-wal.1733192428354.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733192432813 (+8 ms)Rename recovered edits hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/recovered.edits/0000000000000000003-wal.1733192428354.temp to hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/recovered.edits/0000000000000000008 at 1733192432815 (+2 ms)Processed 5 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:44855/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354, size=0, length=0, corrupted=false, cancelled=false at 1733192432816 (+1 ms) 2024-12-03T02:20:32,819 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44855/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354 to hdfs://localhost:44855/hbase/oldWALs/wal.1733192428354 2024-12-03T02:20:32,821 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/recovered.edits/0000000000000000008 2024-12-03T02:20:32,821 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:32,823 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:32,836 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192432824, exclude list is [], retry=0 2024-12-03T02:20:32,839 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:32,840 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:32,840 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:32,845 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192432824 2024-12-03T02:20:32,846 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:36675:36675)] 2024-12-03T02:20:32,846 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d7ca1d81f9ffdce3fe685e573257560, NAME => 'testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:32,846 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:32,847 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:32,847 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:32,852 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:32,853 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d7ca1d81f9ffdce3fe685e573257560 columnFamilyName a 2024-12-03T02:20:32,853 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:32,862 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/194ff877782f4b66b370cd57463b3d21 2024-12-03T02:20:32,867 DEBUG [StoreFileOpener-2d7ca1d81f9ffdce3fe685e573257560-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1bd7759c8aa24cc89d3c35d8708fa8cc_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T02:20:32,867 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1bd7759c8aa24cc89d3c35d8708fa8cc_SeqId_4_ 2024-12-03T02:20:32,871 DEBUG [StoreFileOpener-2d7ca1d81f9ffdce3fe685e573257560-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1ef26df5f5f044b9a477c72197e8e89b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T02:20:32,871 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1ef26df5f5f044b9a477c72197e8e89b_SeqId_4_ 2024-12-03T02:20:32,875 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/d7f20d27229b42b09d0c099ff899847c 2024-12-03T02:20:32,879 DEBUG [StoreFileOpener-2d7ca1d81f9ffdce3fe685e573257560-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for f4ea0a5ffcdb4d76819b5870561d027a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T02:20:32,879 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/f4ea0a5ffcdb4d76819b5870561d027a_SeqId_4_ 2024-12-03T02:20:32,879 WARN [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/194ff877782f4b66b370cd57463b3d21 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2df6473c 2024-12-03T02:20:32,879 WARN [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1bd7759c8aa24cc89d3c35d8708fa8cc_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2df6473c 2024-12-03T02:20:32,880 WARN [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1ef26df5f5f044b9a477c72197e8e89b_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2df6473c 2024-12-03T02:20:32,880 WARN [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/f4ea0a5ffcdb4d76819b5870561d027a_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2df6473c 2024-12-03T02:20:32,880 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/194ff877782f4b66b370cd57463b3d21, hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1bd7759c8aa24cc89d3c35d8708fa8cc_SeqId_4_, hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1ef26df5f5f044b9a477c72197e8e89b_SeqId_4_, hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/f4ea0a5ffcdb4d76819b5870561d027a_SeqId_4_] to archive 2024-12-03T02:20:32,881 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:20:32,883 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/194ff877782f4b66b370cd57463b3d21 to hdfs://localhost:44855/hbase/archive/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/194ff877782f4b66b370cd57463b3d21 2024-12-03T02:20:32,885 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1bd7759c8aa24cc89d3c35d8708fa8cc_SeqId_4_ to hdfs://localhost:44855/hbase/archive/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1bd7759c8aa24cc89d3c35d8708fa8cc_SeqId_4_ 2024-12-03T02:20:32,886 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1ef26df5f5f044b9a477c72197e8e89b_SeqId_4_ to hdfs://localhost:44855/hbase/archive/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/1ef26df5f5f044b9a477c72197e8e89b_SeqId_4_ 2024-12-03T02:20:32,887 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/f4ea0a5ffcdb4d76819b5870561d027a_SeqId_4_ to hdfs://localhost:44855/hbase/archive/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/a/f4ea0a5ffcdb4d76819b5870561d027a_SeqId_4_ 2024-12-03T02:20:32,888 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(327): Store=2d7ca1d81f9ffdce3fe685e573257560/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:32,888 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:32,889 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d7ca1d81f9ffdce3fe685e573257560 columnFamilyName b 2024-12-03T02:20:32,889 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:32,890 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(327): Store=2d7ca1d81f9ffdce3fe685e573257560/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:32,890 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:32,891 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d7ca1d81f9ffdce3fe685e573257560 columnFamilyName c 2024-12-03T02:20:32,891 DEBUG [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:32,892 INFO [StoreOpener-2d7ca1d81f9ffdce3fe685e573257560-1 {}] regionserver.HStore(327): Store=2d7ca1d81f9ffdce3fe685e573257560/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:32,892 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:32,893 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:32,895 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:32,895 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/recovered.edits/0000000000000000008 2024-12-03T02:20:32,900 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 2d7ca1d81f9ffdce3fe685e573257560 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "2d7ca1d81f9ffdce3fe685e573257560" family_name: "a" compaction_input: "194ff877782f4b66b370cd57463b3d21" compaction_input: "1ef26df5f5f044b9a477c72197e8e89b_SeqId_4_" compaction_input: "f4ea0a5ffcdb4d76819b5870561d027a_SeqId_4_" compaction_input: "1bd7759c8aa24cc89d3c35d8708fa8cc_SeqId_4_" compaction_output: "d7f20d27229b42b09d0c099ff899847c" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-03T02:20:32,900 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-03T02:20:32,900 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/recovered.edits/0000000000000000008 2024-12-03T02:20:32,901 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/recovered.edits/0000000000000000008 2024-12-03T02:20:32,903 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:32,903 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:32,904 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:32,906 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2d7ca1d81f9ffdce3fe685e573257560 2024-12-03T02:20:32,909 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testCompactedBulkLoadedFiles/2d7ca1d81f9ffdce3fe685e573257560/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-03T02:20:32,910 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2d7ca1d81f9ffdce3fe685e573257560; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64246037, jitterRate=-0.042659446597099304}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:32,911 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2d7ca1d81f9ffdce3fe685e573257560: Writing region info on filesystem at 1733192432847Initializing all the Stores at 1733192432848 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192432848Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192432852 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192432852Cleaning up temporary data from old regions at 1733192432903 (+51 ms)Region opened successfully at 1733192432911 (+8 ms) 2024-12-03T02:20:32,914 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2d7ca1d81f9ffdce3fe685e573257560, disabling compactions & flushes 2024-12-03T02:20:32,914 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560. 2024-12-03T02:20:32,914 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560. 2024-12-03T02:20:32,914 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560. after waiting 0 ms 2024-12-03T02:20:32,914 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560. 2024-12-03T02:20:32,915 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733192428246.2d7ca1d81f9ffdce3fe685e573257560. 2024-12-03T02:20:32,915 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2d7ca1d81f9ffdce3fe685e573257560: Waiting for close lock at 1733192432914Disabling compacts and flushes for region at 1733192432914Disabling writes for close at 1733192432914Writing region close event to WAL at 1733192432915 (+1 ms)Closed at 1733192432915 2024-12-03T02:20:32,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741924_1103 (size=93) 2024-12-03T02:20:32,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741924_1103 (size=93) 2024-12-03T02:20:32,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741924_1103 (size=93) 2024-12-03T02:20:32,923 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T02:20:32,924 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733192432824) 2024-12-03T02:20:32,937 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=444 (was 428) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1797693541_22 at /127.0.0.1:33030 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1162761534) connection to localhost/127.0.0.1:44855 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1797693541_22 at /127.0.0.1:45662 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:44855 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1797693541_22 at /127.0.0.1:59248 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1260) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=391 (was 425), ProcessCount=11 (was 11), AvailableMemoryMB=242 (was 263) 2024-12-03T02:20:32,937 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-03T02:20:32,948 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=444, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=391, ProcessCount=11, AvailableMemoryMB=242 2024-12-03T02:20:32,948 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-03T02:20:32,961 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:32,963 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:20:32,964 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:20:32,966 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-65390543, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/hregion-65390543, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:32,978 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-65390543/hregion-65390543.1733192432966, exclude list is [], retry=0 2024-12-03T02:20:32,980 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:32,980 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:32,981 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:32,982 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-65390543/hregion-65390543.1733192432966 2024-12-03T02:20:32,983 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:36675:36675)] 2024-12-03T02:20:32,983 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 90ba949a64a4b8a6a0b07f6f0225aa7a, NAME => 'testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44855/hbase 2024-12-03T02:20:32,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741926_1105 (size=67) 2024-12-03T02:20:32,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741926_1105 (size=67) 2024-12-03T02:20:32,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741926_1105 (size=67) 2024-12-03T02:20:32,992 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:32,993 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:32,994 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90ba949a64a4b8a6a0b07f6f0225aa7a columnFamilyName a 2024-12-03T02:20:32,994 DEBUG [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:32,995 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(327): Store=90ba949a64a4b8a6a0b07f6f0225aa7a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:32,995 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:32,996 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90ba949a64a4b8a6a0b07f6f0225aa7a columnFamilyName b 2024-12-03T02:20:32,996 DEBUG [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:32,997 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(327): Store=90ba949a64a4b8a6a0b07f6f0225aa7a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:32,997 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:32,998 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90ba949a64a4b8a6a0b07f6f0225aa7a columnFamilyName c 2024-12-03T02:20:32,998 DEBUG [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:32,999 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(327): Store=90ba949a64a4b8a6a0b07f6f0225aa7a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:32,999 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,000 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,000 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,001 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,001 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,002 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:33,003 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,005 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:20:33,006 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 90ba949a64a4b8a6a0b07f6f0225aa7a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74839439, jitterRate=0.11519454419612885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:33,006 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 90ba949a64a4b8a6a0b07f6f0225aa7a: Writing region info on filesystem at 1733192432992Initializing all the Stores at 1733192432993 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192432993Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192432993Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192432993Cleaning up temporary data from old regions at 1733192433001 (+8 ms)Region opened successfully at 1733192433006 (+5 ms) 2024-12-03T02:20:33,006 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 90ba949a64a4b8a6a0b07f6f0225aa7a, disabling compactions & flushes 2024-12-03T02:20:33,006 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. 2024-12-03T02:20:33,006 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. 2024-12-03T02:20:33,006 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. after waiting 0 ms 2024-12-03T02:20:33,007 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. 2024-12-03T02:20:33,007 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. 2024-12-03T02:20:33,007 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 90ba949a64a4b8a6a0b07f6f0225aa7a: Waiting for close lock at 1733192433006Disabling compacts and flushes for region at 1733192433006Disabling writes for close at 1733192433007 (+1 ms)Writing region close event to WAL at 1733192433007Closed at 1733192433007 2024-12-03T02:20:33,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741925_1104 (size=93) 2024-12-03T02:20:33,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741925_1104 (size=93) 2024-12-03T02:20:33,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741925_1104 (size=93) 2024-12-03T02:20:33,013 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T02:20:33,013 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-65390543:(num 1733192432966) 2024-12-03T02:20:33,014 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:33,016 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:33,028 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433016, exclude list is [], retry=0 2024-12-03T02:20:33,031 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:33,031 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:33,031 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:33,034 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433016 2024-12-03T02:20:33,034 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:36675:36675)] 2024-12-03T02:20:33,034 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 90ba949a64a4b8a6a0b07f6f0225aa7a, NAME => 'testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:33,034 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:33,034 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,034 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,036 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,036 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90ba949a64a4b8a6a0b07f6f0225aa7a columnFamilyName a 2024-12-03T02:20:33,036 DEBUG [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:33,037 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(327): Store=90ba949a64a4b8a6a0b07f6f0225aa7a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:33,037 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,038 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90ba949a64a4b8a6a0b07f6f0225aa7a columnFamilyName b 2024-12-03T02:20:33,038 DEBUG [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:33,039 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(327): Store=90ba949a64a4b8a6a0b07f6f0225aa7a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:33,039 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,040 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90ba949a64a4b8a6a0b07f6f0225aa7a columnFamilyName c 2024-12-03T02:20:33,040 DEBUG [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:33,040 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(327): Store=90ba949a64a4b8a6a0b07f6f0225aa7a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:33,040 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,041 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,042 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,043 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,044 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,044 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:33,045 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,046 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 90ba949a64a4b8a6a0b07f6f0225aa7a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60769046, jitterRate=-0.09447064995765686}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:33,046 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 90ba949a64a4b8a6a0b07f6f0225aa7a: Writing region info on filesystem at 1733192433034Initializing all the Stores at 1733192433035 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192433035Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192433036 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192433036Cleaning up temporary data from old regions at 1733192433044 (+8 ms)Region opened successfully at 1733192433046 (+2 ms) 2024-12-03T02:20:33,054 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-03T02:20:33,068 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/a/e70d22aa49154f41b7b94407f6d1d257 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733192433046/Put/seqid=0 2024-12-03T02:20:33,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741928_1107 (size=5958) 2024-12-03T02:20:33,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741928_1107 (size=5958) 2024-12-03T02:20:33,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741928_1107 (size=5958) 2024-12-03T02:20:33,076 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/a/e70d22aa49154f41b7b94407f6d1d257 2024-12-03T02:20:33,083 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/a/e70d22aa49154f41b7b94407f6d1d257 as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/a/e70d22aa49154f41b7b94407f6d1d257 2024-12-03T02:20:33,089 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/a/e70d22aa49154f41b7b94407f6d1d257, entries=10, sequenceid=13, filesize=5.8 K 2024-12-03T02:20:33,091 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 90ba949a64a4b8a6a0b07f6f0225aa7a in 37ms, sequenceid=13, compaction requested=false 2024-12-03T02:20:33,091 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 90ba949a64a4b8a6a0b07f6f0225aa7a: 2024-12-03T02:20:33,112 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 90ba949a64a4b8a6a0b07f6f0225aa7a, disabling compactions & flushes 2024-12-03T02:20:33,112 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. 2024-12-03T02:20:33,112 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. 2024-12-03T02:20:33,112 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. after waiting 0 ms 2024-12-03T02:20:33,112 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. 2024-12-03T02:20:33,113 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. 2024-12-03T02:20:33,113 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. 2024-12-03T02:20:33,113 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 90ba949a64a4b8a6a0b07f6f0225aa7a: Waiting for close lock at 1733192433112Disabling compacts and flushes for region at 1733192433112Disabling writes for close at 1733192433112Writing region close event to WAL at 1733192433113 (+1 ms)Closed at 1733192433113 2024-12-03T02:20:33,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741927_1106 (size=5869) 2024-12-03T02:20:33,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741927_1106 (size=5869) 2024-12-03T02:20:33,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741927_1106 (size=5869) 2024-12-03T02:20:33,129 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433016, size=5.7 K (5869bytes) 2024-12-03T02:20:33,130 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433016 2024-12-03T02:20:33,130 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433016 after 0ms 2024-12-03T02:20:33,132 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433016 took 3ms 2024-12-03T02:20:33,134 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433016 so closing down 2024-12-03T02:20:33,134 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T02:20:33,135 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733192433016.temp 2024-12-03T02:20:33,136 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000003-wal.1733192433016.temp 2024-12-03T02:20:33,136 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T02:20:33,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741929_1108 (size=5223) 2024-12-03T02:20:33,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741929_1108 (size=5223) 2024-12-03T02:20:33,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741929_1108 (size=5223) 2024-12-03T02:20:33,142 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000003-wal.1733192433016.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T02:20:33,143 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000003-wal.1733192433016.temp to hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000035 2024-12-03T02:20:33,143 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 10 ms; skipped=2; WAL=hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433016, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-12-03T02:20:33,144 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433016, journal: Splitting hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433016, size=5.7 K (5869bytes) at 1733192433130Finishing writing output for hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433016 so closing down at 1733192433134 (+4 ms)Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000003-wal.1733192433016.temp at 1733192433136 (+2 ms)3 split writer threads finished at 1733192433136Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000003-wal.1733192433016.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733192433142 (+6 ms)Rename recovered edits hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000003-wal.1733192433016.temp to hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000035 at 1733192433143 (+1 ms)Processed 32 edits across 1 Regions in 10 ms; skipped=2; WAL=hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433016, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1733192433143 2024-12-03T02:20:33,145 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433016 to hdfs://localhost:44855/hbase/oldWALs/wal.1733192433016 2024-12-03T02:20:33,146 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000035 2024-12-03T02:20:33,146 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:33,147 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:33,158 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147, exclude list is [], retry=0 2024-12-03T02:20:33,161 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:33,161 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:33,161 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:33,163 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147 2024-12-03T02:20:33,163 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36675:36675),(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:39215:39215)] 2024-12-03T02:20:33,163 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 90ba949a64a4b8a6a0b07f6f0225aa7a, NAME => 'testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:20:33,163 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:33,164 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,164 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,165 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,166 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90ba949a64a4b8a6a0b07f6f0225aa7a columnFamilyName a 2024-12-03T02:20:33,166 DEBUG [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:33,171 DEBUG [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/a/e70d22aa49154f41b7b94407f6d1d257 2024-12-03T02:20:33,171 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(327): Store=90ba949a64a4b8a6a0b07f6f0225aa7a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:33,171 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,172 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90ba949a64a4b8a6a0b07f6f0225aa7a columnFamilyName b 2024-12-03T02:20:33,172 DEBUG [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:33,172 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(327): Store=90ba949a64a4b8a6a0b07f6f0225aa7a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:33,173 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,173 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90ba949a64a4b8a6a0b07f6f0225aa7a columnFamilyName c 2024-12-03T02:20:33,173 DEBUG [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:33,174 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(327): Store=90ba949a64a4b8a6a0b07f6f0225aa7a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:33,174 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,174 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,176 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,176 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000035 2024-12-03T02:20:33,179 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000035 2024-12-03T02:20:33,179 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-03T02:20:33,196 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/b/03029b037fdc479e898537a217e24ae5 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733192433091/Put/seqid=0 2024-12-03T02:20:33,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741931_1110 (size=5958) 2024-12-03T02:20:33,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741931_1110 (size=5958) 2024-12-03T02:20:33,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741931_1110 (size=5958) 2024-12-03T02:20:33,203 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/b/03029b037fdc479e898537a217e24ae5 2024-12-03T02:20:33,227 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/c/05811ab7877a4e5c952be21415b08701 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733192433101/Put/seqid=0 2024-12-03T02:20:33,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741932_1111 (size=5958) 2024-12-03T02:20:33,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741932_1111 (size=5958) 2024-12-03T02:20:33,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741932_1111 (size=5958) 2024-12-03T02:20:33,235 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/c/05811ab7877a4e5c952be21415b08701 2024-12-03T02:20:33,241 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/b/03029b037fdc479e898537a217e24ae5 as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/b/03029b037fdc479e898537a217e24ae5 2024-12-03T02:20:33,246 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/b/03029b037fdc479e898537a217e24ae5, entries=10, sequenceid=35, filesize=5.8 K 2024-12-03T02:20:33,247 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/c/05811ab7877a4e5c952be21415b08701 as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/c/05811ab7877a4e5c952be21415b08701 2024-12-03T02:20:33,252 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/c/05811ab7877a4e5c952be21415b08701, entries=10, sequenceid=35, filesize=5.8 K 2024-12-03T02:20:33,252 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 90ba949a64a4b8a6a0b07f6f0225aa7a in 73ms, sequenceid=35, compaction requested=false; wal=null 2024-12-03T02:20:33,253 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000035 2024-12-03T02:20:33,254 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,254 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,254 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:33,256 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:33,257 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-03T02:20:33,258 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 90ba949a64a4b8a6a0b07f6f0225aa7a; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72766344, jitterRate=0.08430302143096924}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:33,258 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 90ba949a64a4b8a6a0b07f6f0225aa7a: Writing region info on filesystem at 1733192433164Initializing all the Stores at 1733192433165 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192433165Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192433165Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192433165Obtaining lock to block concurrent updates at 1733192433179 (+14 ms)Preparing flush snapshotting stores in 90ba949a64a4b8a6a0b07f6f0225aa7a at 1733192433179Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733192433179Flushing stores of testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. at 1733192433179Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a/b: creating writer at 1733192433180 (+1 ms)Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a/b: appending metadata at 1733192433195 (+15 ms)Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a/b: closing flushed file at 1733192433195Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a/c: creating writer at 1733192433207 (+12 ms)Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a/c: appending metadata at 1733192433226 (+19 ms)Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a/c: closing flushed file at 1733192433226Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@443ed5f2: reopening flushed file at 1733192433240 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70e4a59e: reopening flushed file at 1733192433246 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 90ba949a64a4b8a6a0b07f6f0225aa7a in 73ms, sequenceid=35, compaction requested=false; wal=null at 1733192433252 (+6 ms)Cleaning up temporary data from old regions at 1733192433254 (+2 ms)Region opened successfully at 1733192433258 (+4 ms) 2024-12-03T02:20:33,319 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147, size=0 (0bytes) 2024-12-03T02:20:33,319 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147 might be still open, length is 0 2024-12-03T02:20:33,319 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147 2024-12-03T02:20:33,320 WARN [IPC Server handler 3 on default port 44855 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-03T02:20:33,320 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147 after 1ms 2024-12-03T02:20:34,768 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:33070 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:35125:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33070 dst: /127.0.0.1:35125 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35125 remote=/127.0.0.1:33070]. Total timeout mills is 60000, 58516 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:34,768 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:45724 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:43037:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45724 dst: /127.0.0.1:43037 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:34,768 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1610019660_22 at /127.0.0.1:59320 [Receiving block BP-537057715-172.17.0.3-1733192391930:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:36885:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59320 dst: /127.0.0.1:36885 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:34,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741930_1112 (size=5215) 2024-12-03T02:20:34,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741930_1112 (size=5215) 2024-12-03T02:20:37,043 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-03T02:20:37,043 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-03T02:20:37,044 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-03T02:20:37,044 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-03T02:20:37,045 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-03T02:20:37,045 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-03T02:20:37,321 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147 after 4002ms 2024-12-03T02:20:37,324 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147 took 4005ms 2024-12-03T02:20:37,326 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147; continuing. 2024-12-03T02:20:37,326 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147 so closing down 2024-12-03T02:20:37,326 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T02:20:37,328 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733192433147.temp 2024-12-03T02:20:37,330 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000037-wal.1733192433147.temp 2024-12-03T02:20:37,331 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T02:20:37,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741933_1113 (size=5223) 2024-12-03T02:20:37,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741933_1113 (size=5223) 2024-12-03T02:20:37,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741933_1113 (size=5223) 2024-12-03T02:20:37,769 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000037-wal.1733192433147.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T02:20:37,773 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000037-wal.1733192433147.temp to hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000066 2024-12-03T02:20:37,774 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 448 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T02:20:37,774 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147, journal: Splitting hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147, size=0 (0bytes) at 1733192433319Finishing writing output for hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147 so closing down at 1733192437326 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000037-wal.1733192433147.temp at 1733192437330 (+4 ms)3 split writer threads finished at 1733192437331 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000037-wal.1733192433147.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733192437770 (+439 ms)Rename recovered edits hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000037-wal.1733192433147.temp to hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000066 at 1733192437773 (+3 ms)Processed 30 edits across 1 Regions in 448 ms; skipped=0; WAL=hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147, size=0, length=0, corrupted=false, cancelled=false at 1733192437774 (+1 ms) 2024-12-03T02:20:37,777 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147 to hdfs://localhost:44855/hbase/oldWALs/wal.1733192433147 2024-12-03T02:20:37,778 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000066 2024-12-03T02:20:37,778 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T02:20:37,780 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44855/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960, archiveDir=hdfs://localhost:44855/hbase/oldWALs, maxLogs=32 2024-12-03T02:20:37,794 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192437781, exclude list is [], retry=0 2024-12-03T02:20:37,797 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43037,DS-01018773-b73a-481d-8884-49ff5e2559e9,DISK] 2024-12-03T02:20:37,797 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36885,DS-51edcd96-2d93-432f-b830-e47c1264fe0a,DISK] 2024-12-03T02:20:37,798 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-d96ac299-498a-44ac-b8c3-96f81fc5aa7f,DISK] 2024-12-03T02:20:37,799 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192437781 2024-12-03T02:20:37,800 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43847:43847),(127.0.0.1/127.0.0.1:39215:39215),(127.0.0.1/127.0.0.1:36675:36675)] 2024-12-03T02:20:37,800 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:20:37,802 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:37,803 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90ba949a64a4b8a6a0b07f6f0225aa7a columnFamilyName a 2024-12-03T02:20:37,803 DEBUG [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:37,811 DEBUG [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/a/e70d22aa49154f41b7b94407f6d1d257 2024-12-03T02:20:37,812 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(327): Store=90ba949a64a4b8a6a0b07f6f0225aa7a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:37,812 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:37,813 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90ba949a64a4b8a6a0b07f6f0225aa7a columnFamilyName b 2024-12-03T02:20:37,813 DEBUG [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:37,819 DEBUG [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/b/03029b037fdc479e898537a217e24ae5 2024-12-03T02:20:37,819 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(327): Store=90ba949a64a4b8a6a0b07f6f0225aa7a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:37,819 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:37,820 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90ba949a64a4b8a6a0b07f6f0225aa7a columnFamilyName c 2024-12-03T02:20:37,820 DEBUG [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:20:37,841 DEBUG [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/c/05811ab7877a4e5c952be21415b08701 2024-12-03T02:20:37,842 INFO [StoreOpener-90ba949a64a4b8a6a0b07f6f0225aa7a-1 {}] regionserver.HStore(327): Store=90ba949a64a4b8a6a0b07f6f0225aa7a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:20:37,842 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:37,843 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:37,845 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:37,845 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000066 2024-12-03T02:20:37,852 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000066 2024-12-03T02:20:37,852 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-03T02:20:37,869 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/a/e9d3a85b0ffb413db44b307b83afca9a is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733192433266/Put/seqid=0 2024-12-03T02:20:37,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741935_1115 (size=5958) 2024-12-03T02:20:37,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741935_1115 (size=5958) 2024-12-03T02:20:37,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741935_1115 (size=5958) 2024-12-03T02:20:37,877 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/a/e9d3a85b0ffb413db44b307b83afca9a 2024-12-03T02:20:37,895 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/b/4011930187564f43b47014dffebdaf27 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733192433273/Put/seqid=0 2024-12-03T02:20:37,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741936_1116 (size=5958) 2024-12-03T02:20:37,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741936_1116 (size=5958) 2024-12-03T02:20:37,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741936_1116 (size=5958) 2024-12-03T02:20:37,901 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/b/4011930187564f43b47014dffebdaf27 2024-12-03T02:20:37,918 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/c/74a45acd2d194c62af24e6d89d2a8419 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733192433279/Put/seqid=0 2024-12-03T02:20:37,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741937_1117 (size=5958) 2024-12-03T02:20:37,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741937_1117 (size=5958) 2024-12-03T02:20:37,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741937_1117 (size=5958) 2024-12-03T02:20:37,925 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/c/74a45acd2d194c62af24e6d89d2a8419 2024-12-03T02:20:37,929 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/a/e9d3a85b0ffb413db44b307b83afca9a as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/a/e9d3a85b0ffb413db44b307b83afca9a 2024-12-03T02:20:37,933 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/a/e9d3a85b0ffb413db44b307b83afca9a, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T02:20:37,935 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/b/4011930187564f43b47014dffebdaf27 as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/b/4011930187564f43b47014dffebdaf27 2024-12-03T02:20:37,940 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/b/4011930187564f43b47014dffebdaf27, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T02:20:37,941 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/.tmp/c/74a45acd2d194c62af24e6d89d2a8419 as hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/c/74a45acd2d194c62af24e6d89d2a8419 2024-12-03T02:20:37,945 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/c/74a45acd2d194c62af24e6d89d2a8419, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T02:20:37,946 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 90ba949a64a4b8a6a0b07f6f0225aa7a in 94ms, sequenceid=66, compaction requested=false; wal=null 2024-12-03T02:20:37,946 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/0000000000000000066 2024-12-03T02:20:37,947 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:37,948 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:37,948 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:20:37,950 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 90ba949a64a4b8a6a0b07f6f0225aa7a 2024-12-03T02:20:37,952 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/hbase/data/default/testReplayEditsWrittenViaHRegion/90ba949a64a4b8a6a0b07f6f0225aa7a/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-03T02:20:37,954 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 90ba949a64a4b8a6a0b07f6f0225aa7a; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68596377, jitterRate=0.02216567099094391}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:20:37,954 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 90ba949a64a4b8a6a0b07f6f0225aa7a: Writing region info on filesystem at 1733192437800Initializing all the Stores at 1733192437802 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192437802Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192437802Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192437802Obtaining lock to block concurrent updates at 1733192437852 (+50 ms)Preparing flush snapshotting stores in 90ba949a64a4b8a6a0b07f6f0225aa7a at 1733192437852Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733192437852Flushing stores of testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. at 1733192437852Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a/a: creating writer at 1733192437852Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a/a: appending metadata at 1733192437868 (+16 ms)Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a/a: closing flushed file at 1733192437868Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a/b: creating writer at 1733192437881 (+13 ms)Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a/b: appending metadata at 1733192437894 (+13 ms)Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a/b: closing flushed file at 1733192437894Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a/c: creating writer at 1733192437906 (+12 ms)Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a/c: appending metadata at 1733192437918 (+12 ms)Flushing 90ba949a64a4b8a6a0b07f6f0225aa7a/c: closing flushed file at 1733192437918Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@723a6bdb: reopening flushed file at 1733192437928 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39a4d7ba: reopening flushed file at 1733192437934 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f5b1277: reopening flushed file at 1733192437940 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 90ba949a64a4b8a6a0b07f6f0225aa7a in 94ms, sequenceid=66, compaction requested=false; wal=null at 1733192437946 (+6 ms)Cleaning up temporary data from old regions at 1733192437948 (+2 ms)Region opened successfully at 1733192437954 (+6 ms) 2024-12-03T02:20:37,967 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 90ba949a64a4b8a6a0b07f6f0225aa7a, disabling compactions & flushes 2024-12-03T02:20:37,967 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. 2024-12-03T02:20:37,967 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. 2024-12-03T02:20:37,967 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. after waiting 0 ms 2024-12-03T02:20:37,967 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. 2024-12-03T02:20:37,968 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733192432962.90ba949a64a4b8a6a0b07f6f0225aa7a. 2024-12-03T02:20:37,968 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 90ba949a64a4b8a6a0b07f6f0225aa7a: Waiting for close lock at 1733192437967Disabling compacts and flushes for region at 1733192437967Disabling writes for close at 1733192437967Writing region close event to WAL at 1733192437968 (+1 ms)Closed at 1733192437968 2024-12-03T02:20:37,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741934_1114 (size=93) 2024-12-03T02:20:37,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741934_1114 (size=93) 2024-12-03T02:20:37,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741934_1114 (size=93) 2024-12-03T02:20:37,973 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T02:20:37,973 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733192437781) 2024-12-03T02:20:37,986 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=449 (was 444) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1297624713_22 at /127.0.0.1:46780 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:44855 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1297624713_22 at /127.0.0.1:38334 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1162761534) connection to localhost/127.0.0.1:44855 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1410 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 391), ProcessCount=11 (was 11), AvailableMemoryMB=226 (was 242) 2024-12-03T02:20:37,986 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1410 is superior to 1024 2024-12-03T02:20:37,986 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T02:20:37,987 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T02:20:37,987 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:20:37,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:20:37,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:20:37,988 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:20:37,988 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T02:20:37,988 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=750567030, stopped=false 2024-12-03T02:20:37,988 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=64035cbb0b46,33741,1733192396696 2024-12-03T02:20:38,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T02:20:38,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T02:20:38,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T02:20:38,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:20:38,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:20:38,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:20:38,012 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T02:20:38,012 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T02:20:38,012 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T02:20:38,013 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T02:20:38,013 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:20:38,013 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T02:20:38,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:20:38,013 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '64035cbb0b46,44467,1733192397576' ***** 2024-12-03T02:20:38,013 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T02:20:38,013 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '64035cbb0b46,39333,1733192397619' ***** 2024-12-03T02:20:38,013 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T02:20:38,013 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T02:20:38,014 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T02:20:38,014 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T02:20:38,014 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T02:20:38,014 INFO [RS:2;64035cbb0b46:39333 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T02:20:38,014 INFO [RS:1;64035cbb0b46:44467 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T02:20:38,014 INFO [RS:2;64035cbb0b46:39333 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T02:20:38,014 INFO [RS:1;64035cbb0b46:44467 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T02:20:38,014 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(3091): Received CLOSE for 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:38,014 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(959): stopping server 64035cbb0b46,44467,1733192397576 2024-12-03T02:20:38,014 INFO [RS:1;64035cbb0b46:44467 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T02:20:38,014 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(959): stopping server 64035cbb0b46,39333,1733192397619 2024-12-03T02:20:38,014 INFO [RS:1;64035cbb0b46:44467 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;64035cbb0b46:44467. 2024-12-03T02:20:38,014 INFO [RS:2;64035cbb0b46:39333 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T02:20:38,014 INFO [RS:2;64035cbb0b46:39333 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;64035cbb0b46:39333. 2024-12-03T02:20:38,014 DEBUG [RS:1;64035cbb0b46:44467 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:20:38,014 DEBUG [RS:2;64035cbb0b46:39333 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:20:38,014 DEBUG [RS:1;64035cbb0b46:44467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:20:38,014 DEBUG [RS:2;64035cbb0b46:39333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:20:38,014 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 63293c0fcf6d2a20b48444c1ca6e401a, disabling compactions & flushes 2024-12-03T02:20:38,015 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T02:20:38,015 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T02:20:38,015 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:38,015 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T02:20:38,015 DEBUG [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(1325): Online Regions={63293c0fcf6d2a20b48444c1ca6e401a=testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a.} 2024-12-03T02:20:38,015 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T02:20:38,015 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:38,015 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. after waiting 0 ms 2024-12-03T02:20:38,015 DEBUG [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(1351): Waiting on 63293c0fcf6d2a20b48444c1ca6e401a 2024-12-03T02:20:38,015 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:38,015 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T02:20:38,015 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T02:20:38,015 DEBUG [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-03T02:20:38,015 DEBUG [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T02:20:38,015 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T02:20:38,015 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T02:20:38,015 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T02:20:38,015 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T02:20:38,015 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T02:20:38,015 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-03T02:20:38,019 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/63293c0fcf6d2a20b48444c1ca6e401a/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-03T02:20:38,020 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:38,020 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 63293c0fcf6d2a20b48444c1ca6e401a: Waiting for close lock at 1733192438014Running coprocessor pre-close hooks at 1733192438014Disabling compacts and flushes for region at 1733192438014Disabling writes for close at 1733192438015 (+1 ms)Writing region close event to WAL at 1733192438016 (+1 ms)Running coprocessor post-close hooks at 1733192438020 (+4 ms)Closed at 1733192438020 2024-12-03T02:20:38,020 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a. 2024-12-03T02:20:38,040 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740/.tmp/info/53480279992647a99f4cd88fd9480afd is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733192412829.63293c0fcf6d2a20b48444c1ca6e401a./info:regioninfo/1733192416218/Put/seqid=0 2024-12-03T02:20:38,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741938_1118 (size=8243) 2024-12-03T02:20:38,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741938_1118 (size=8243) 2024-12-03T02:20:38,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741938_1118 (size=8243) 2024-12-03T02:20:38,046 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740/.tmp/info/53480279992647a99f4cd88fd9480afd 2024-12-03T02:20:38,049 INFO [regionserver/64035cbb0b46:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T02:20:38,063 INFO [regionserver/64035cbb0b46:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T02:20:38,066 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740/.tmp/ns/865c2f4ca29b4da0819ed675d2829a27 is 43, key is default/ns:d/1733192400002/Put/seqid=0 2024-12-03T02:20:38,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741939_1119 (size=5153) 2024-12-03T02:20:38,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741939_1119 (size=5153) 2024-12-03T02:20:38,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741939_1119 (size=5153) 2024-12-03T02:20:38,073 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740/.tmp/ns/865c2f4ca29b4da0819ed675d2829a27 2024-12-03T02:20:38,111 INFO [regionserver/64035cbb0b46:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T02:20:38,111 INFO [regionserver/64035cbb0b46:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T02:20:38,111 INFO [regionserver/64035cbb0b46:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T02:20:38,111 INFO [regionserver/64035cbb0b46:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T02:20:38,115 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740/.tmp/table/f79d2271d7004f50ac330b1974882d25 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733192413292/Put/seqid=0 2024-12-03T02:20:38,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741940_1120 (size=5431) 2024-12-03T02:20:38,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741940_1120 (size=5431) 2024-12-03T02:20:38,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741940_1120 (size=5431) 2024-12-03T02:20:38,123 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740/.tmp/table/f79d2271d7004f50ac330b1974882d25 2024-12-03T02:20:38,129 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740/.tmp/info/53480279992647a99f4cd88fd9480afd as hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740/info/53480279992647a99f4cd88fd9480afd 2024-12-03T02:20:38,135 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740/info/53480279992647a99f4cd88fd9480afd, entries=18, sequenceid=21, filesize=8.0 K 2024-12-03T02:20:38,136 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740/.tmp/ns/865c2f4ca29b4da0819ed675d2829a27 as hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740/ns/865c2f4ca29b4da0819ed675d2829a27 2024-12-03T02:20:38,141 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740/ns/865c2f4ca29b4da0819ed675d2829a27, entries=2, sequenceid=21, filesize=5.0 K 2024-12-03T02:20:38,142 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740/.tmp/table/f79d2271d7004f50ac330b1974882d25 as hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740/table/f79d2271d7004f50ac330b1974882d25 2024-12-03T02:20:38,148 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740/table/f79d2271d7004f50ac330b1974882d25, entries=2, sequenceid=21, filesize=5.3 K 2024-12-03T02:20:38,149 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 134ms, sequenceid=21, compaction requested=false 2024-12-03T02:20:38,154 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-03T02:20:38,155 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T02:20:38,155 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T02:20:38,155 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733192438015Running coprocessor pre-close hooks at 1733192438015Disabling compacts and flushes for region at 1733192438015Disabling writes for close at 1733192438015Obtaining lock to block concurrent updates at 1733192438015Preparing flush snapshotting stores in 1588230740 at 1733192438015Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733192438016 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733192438016Flushing 1588230740/info: creating writer at 1733192438017 (+1 ms)Flushing 1588230740/info: appending metadata at 1733192438039 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733192438039Flushing 1588230740/ns: creating writer at 1733192438053 (+14 ms)Flushing 1588230740/ns: appending metadata at 1733192438066 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1733192438066Flushing 1588230740/table: creating writer at 1733192438078 (+12 ms)Flushing 1588230740/table: appending metadata at 1733192438114 (+36 ms)Flushing 1588230740/table: closing flushed file at 1733192438114Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73506bf4: reopening flushed file at 1733192438128 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b5c2717: reopening flushed file at 1733192438135 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65a11654: reopening flushed file at 1733192438142 (+7 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 134ms, sequenceid=21, compaction requested=false at 1733192438149 (+7 ms)Writing region close event to WAL at 1733192438151 (+2 ms)Running coprocessor post-close hooks at 1733192438155 (+4 ms)Closed at 1733192438155 2024-12-03T02:20:38,155 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T02:20:38,215 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(976): stopping server 64035cbb0b46,39333,1733192397619; all regions closed. 2024-12-03T02:20:38,215 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(976): stopping server 64035cbb0b46,44467,1733192397576; all regions closed. 2024-12-03T02:20:38,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741834_1010 (size=928) 2024-12-03T02:20:38,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741834_1010 (size=928) 2024-12-03T02:20:38,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741834_1010 (size=928) 2024-12-03T02:20:38,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741836_1012 (size=8899) 2024-12-03T02:20:38,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741836_1012 (size=8899) 2024-12-03T02:20:38,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741836_1012 (size=8899) 2024-12-03T02:20:38,226 DEBUG [RS:1;64035cbb0b46:44467 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/oldWALs 2024-12-03T02:20:38,226 INFO [RS:1;64035cbb0b46:44467 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 64035cbb0b46%2C44467%2C1733192397576.meta:.meta(num 1733192399739) 2024-12-03T02:20:38,226 DEBUG [RS:2;64035cbb0b46:39333 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/oldWALs 2024-12-03T02:20:38,226 INFO [RS:2;64035cbb0b46:39333 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 64035cbb0b46%2C39333%2C1733192397619:(num 1733192399275) 2024-12-03T02:20:38,226 DEBUG [RS:2;64035cbb0b46:39333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:20:38,226 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T02:20:38,226 INFO [RS:2;64035cbb0b46:39333 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T02:20:38,226 INFO [RS:2;64035cbb0b46:39333 {}] hbase.ChoreService(370): Chore service for: regionserver/64035cbb0b46:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-03T02:20:38,226 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T02:20:38,226 INFO [regionserver/64035cbb0b46:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T02:20:38,226 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T02:20:38,226 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T02:20:38,227 INFO [RS:2;64035cbb0b46:39333 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T02:20:38,227 INFO [RS:2;64035cbb0b46:39333 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39333 2024-12-03T02:20:38,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741833_1009 (size=2598) 2024-12-03T02:20:38,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741833_1009 (size=2598) 2024-12-03T02:20:38,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741833_1009 (size=2598) 2024-12-03T02:20:38,231 DEBUG [RS:1;64035cbb0b46:44467 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/oldWALs 2024-12-03T02:20:38,231 INFO [RS:1;64035cbb0b46:44467 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 64035cbb0b46%2C44467%2C1733192397576:(num 1733192399275) 2024-12-03T02:20:38,231 DEBUG [RS:1;64035cbb0b46:44467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:20:38,231 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T02:20:38,231 INFO [RS:1;64035cbb0b46:44467 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T02:20:38,232 INFO [RS:1;64035cbb0b46:44467 {}] hbase.ChoreService(370): Chore service for: regionserver/64035cbb0b46:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-03T02:20:38,232 INFO [RS:1;64035cbb0b46:44467 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T02:20:38,232 INFO [regionserver/64035cbb0b46:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T02:20:38,232 INFO [RS:1;64035cbb0b46:44467 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44467 2024-12-03T02:20:38,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/64035cbb0b46,39333,1733192397619 2024-12-03T02:20:38,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T02:20:38,237 INFO [RS:2;64035cbb0b46:39333 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T02:20:38,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/64035cbb0b46,44467,1733192397576 2024-12-03T02:20:38,245 INFO [RS:1;64035cbb0b46:44467 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T02:20:38,253 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [64035cbb0b46,44467,1733192397576] 2024-12-03T02:20:38,270 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/64035cbb0b46,44467,1733192397576 already deleted, retry=false 2024-12-03T02:20:38,270 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 64035cbb0b46,44467,1733192397576 expired; onlineServers=1 2024-12-03T02:20:38,270 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [64035cbb0b46,39333,1733192397619] 2024-12-03T02:20:38,278 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/64035cbb0b46,39333,1733192397619 already deleted, retry=false 2024-12-03T02:20:38,278 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 64035cbb0b46,39333,1733192397619 expired; onlineServers=0 2024-12-03T02:20:38,278 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '64035cbb0b46,33741,1733192396696' ***** 2024-12-03T02:20:38,279 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T02:20:38,279 INFO [M:0;64035cbb0b46:33741 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T02:20:38,279 INFO [M:0;64035cbb0b46:33741 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T02:20:38,279 DEBUG [M:0;64035cbb0b46:33741 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T02:20:38,279 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T02:20:38,279 DEBUG [M:0;64035cbb0b46:33741 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T02:20:38,279 DEBUG [master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.small.0-1733192398811 {}] cleaner.HFileCleaner(306): Exit Thread[master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.small.0-1733192398811,5,FailOnTimeoutGroup] 2024-12-03T02:20:38,279 DEBUG [master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.large.0-1733192398810 {}] cleaner.HFileCleaner(306): Exit Thread[master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.large.0-1733192398810,5,FailOnTimeoutGroup] 2024-12-03T02:20:38,279 INFO [M:0;64035cbb0b46:33741 {}] hbase.ChoreService(370): Chore service for: master/64035cbb0b46:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T02:20:38,280 INFO [M:0;64035cbb0b46:33741 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T02:20:38,280 DEBUG [M:0;64035cbb0b46:33741 {}] master.HMaster(1795): Stopping service threads 2024-12-03T02:20:38,280 INFO [M:0;64035cbb0b46:33741 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T02:20:38,280 INFO [M:0;64035cbb0b46:33741 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T02:20:38,280 INFO [M:0;64035cbb0b46:33741 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T02:20:38,280 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T02:20:38,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T02:20:38,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:20:38,287 DEBUG [M:0;64035cbb0b46:33741 {}] zookeeper.ZKUtil(347): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T02:20:38,287 WARN [M:0;64035cbb0b46:33741 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T02:20:38,288 INFO [M:0;64035cbb0b46:33741 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/.lastflushedseqids 2024-12-03T02:20:38,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741941_1121 (size=138) 2024-12-03T02:20:38,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741941_1121 (size=138) 2024-12-03T02:20:38,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741941_1121 (size=138) 2024-12-03T02:20:38,307 INFO [M:0;64035cbb0b46:33741 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T02:20:38,307 INFO [M:0;64035cbb0b46:33741 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T02:20:38,307 DEBUG [M:0;64035cbb0b46:33741 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T02:20:38,307 INFO [M:0;64035cbb0b46:33741 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:20:38,307 DEBUG [M:0;64035cbb0b46:33741 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:20:38,307 DEBUG [M:0;64035cbb0b46:33741 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T02:20:38,307 DEBUG [M:0;64035cbb0b46:33741 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:20:38,307 INFO [M:0;64035cbb0b46:33741 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.36 KB heapSize=83.73 KB 2024-12-03T02:20:38,322 DEBUG [M:0;64035cbb0b46:33741 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c5b90f5de1ca4c0c858ff3b3f7b7b2cc is 82, key is hbase:meta,,1/info:regioninfo/1733192399860/Put/seqid=0 2024-12-03T02:20:38,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741942_1122 (size=5672) 2024-12-03T02:20:38,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741942_1122 (size=5672) 2024-12-03T02:20:38,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741942_1122 (size=5672) 2024-12-03T02:20:38,328 INFO [M:0;64035cbb0b46:33741 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c5b90f5de1ca4c0c858ff3b3f7b7b2cc 2024-12-03T02:20:38,353 DEBUG [M:0;64035cbb0b46:33741 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2aec218025a14e0e8f6f1db1d0256476 is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733192413297/Put/seqid=0 2024-12-03T02:20:38,354 INFO [RS:2;64035cbb0b46:39333 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T02:20:38,354 INFO [RS:2;64035cbb0b46:39333 {}] regionserver.HRegionServer(1031): Exiting; stopping=64035cbb0b46,39333,1733192397619; zookeeper connection closed. 2024-12-03T02:20:38,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T02:20:38,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39333-0x1019a46de8c0003, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T02:20:38,354 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@24cf4dcb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@24cf4dcb 2024-12-03T02:20:38,362 INFO [RS:1;64035cbb0b46:44467 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T02:20:38,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T02:20:38,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44467-0x1019a46de8c0002, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T02:20:38,362 INFO [RS:1;64035cbb0b46:44467 {}] regionserver.HRegionServer(1031): Exiting; stopping=64035cbb0b46,44467,1733192397576; zookeeper connection closed. 2024-12-03T02:20:38,362 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@17d786ba {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@17d786ba 2024-12-03T02:20:38,363 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-03T02:20:38,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741943_1123 (size=7756) 2024-12-03T02:20:38,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741943_1123 (size=7756) 2024-12-03T02:20:38,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741943_1123 (size=7756) 2024-12-03T02:20:38,372 INFO [M:0;64035cbb0b46:33741 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.62 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2aec218025a14e0e8f6f1db1d0256476 2024-12-03T02:20:38,378 INFO [M:0;64035cbb0b46:33741 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2aec218025a14e0e8f6f1db1d0256476 2024-12-03T02:20:38,392 DEBUG [M:0;64035cbb0b46:33741 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c993df20084d4a80bff9c3e98f3d36da is 69, key is 64035cbb0b46,39333,1733192397619/rs:state/1733192398934/Put/seqid=0 2024-12-03T02:20:38,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741944_1124 (size=5440) 2024-12-03T02:20:38,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741944_1124 (size=5440) 2024-12-03T02:20:38,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741944_1124 (size=5440) 2024-12-03T02:20:38,398 INFO [M:0;64035cbb0b46:33741 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c993df20084d4a80bff9c3e98f3d36da 2024-12-03T02:20:38,403 INFO [M:0;64035cbb0b46:33741 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c993df20084d4a80bff9c3e98f3d36da 2024-12-03T02:20:38,404 DEBUG [M:0;64035cbb0b46:33741 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c5b90f5de1ca4c0c858ff3b3f7b7b2cc as hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c5b90f5de1ca4c0c858ff3b3f7b7b2cc 2024-12-03T02:20:38,408 INFO [M:0;64035cbb0b46:33741 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c5b90f5de1ca4c0c858ff3b3f7b7b2cc, entries=8, sequenceid=168, filesize=5.5 K 2024-12-03T02:20:38,409 DEBUG [M:0;64035cbb0b46:33741 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2aec218025a14e0e8f6f1db1d0256476 as hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2aec218025a14e0e8f6f1db1d0256476 2024-12-03T02:20:38,414 INFO [M:0;64035cbb0b46:33741 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2aec218025a14e0e8f6f1db1d0256476 2024-12-03T02:20:38,414 INFO [M:0;64035cbb0b46:33741 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2aec218025a14e0e8f6f1db1d0256476, entries=17, sequenceid=168, filesize=7.6 K 2024-12-03T02:20:38,415 DEBUG [M:0;64035cbb0b46:33741 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c993df20084d4a80bff9c3e98f3d36da as hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c993df20084d4a80bff9c3e98f3d36da 2024-12-03T02:20:38,420 INFO [M:0;64035cbb0b46:33741 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c993df20084d4a80bff9c3e98f3d36da 2024-12-03T02:20:38,420 INFO [M:0;64035cbb0b46:33741 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44855/user/jenkins/test-data/b9b4ebe7-0619-440e-6874-657c5260c22c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c993df20084d4a80bff9c3e98f3d36da, entries=3, sequenceid=168, filesize=5.3 K 2024-12-03T02:20:38,421 INFO [M:0;64035cbb0b46:33741 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 114ms, sequenceid=168, compaction requested=false 2024-12-03T02:20:38,423 INFO [M:0;64035cbb0b46:33741 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:20:38,423 DEBUG [M:0;64035cbb0b46:33741 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733192438307Disabling compacts and flushes for region at 1733192438307Disabling writes for close at 1733192438307Obtaining lock to block concurrent updates at 1733192438307Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733192438307Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69996, getHeapSize=85680, getOffHeapSize=0, getCellsCount=195 at 1733192438308 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733192438308Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733192438309 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733192438322 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733192438322Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733192438333 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733192438352 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733192438353 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733192438379 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733192438391 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733192438391Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c06decb: reopening flushed file at 1733192438403 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c29fec1: reopening flushed file at 1733192438408 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49d67ffe: reopening flushed file at 1733192438414 (+6 ms)Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 114ms, sequenceid=168, compaction requested=false at 1733192438421 (+7 ms)Writing region close event to WAL at 1733192438422 (+1 ms)Closed at 1733192438422 2024-12-03T02:20:38,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43037 is added to blk_1073741830_1006 (size=81443) 2024-12-03T02:20:38,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741830_1006 (size=81443) 2024-12-03T02:20:38,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36885 is added to blk_1073741830_1006 (size=81443) 2024-12-03T02:20:38,425 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T02:20:38,426 INFO [M:0;64035cbb0b46:33741 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T02:20:38,426 INFO [M:0;64035cbb0b46:33741 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33741 2024-12-03T02:20:38,426 INFO [M:0;64035cbb0b46:33741 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T02:20:38,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T02:20:38,537 INFO [M:0;64035cbb0b46:33741 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T02:20:38,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33741-0x1019a46de8c0000, quorum=127.0.0.1:57355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T02:20:38,543 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417565 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417565 (inode 16655) Holder DFSClient_NONMAPREDUCE_1610019660_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733192417286/wal.1733192417565 (inode 16655) Holder DFSClient_NONMAPREDUCE_1610019660_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T02:20:38,544 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733192408294/wal.1733192408436 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:38,545 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733192432960/wal.1733192433147 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:38,547 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192426927 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192426927 (inode 16678) Holder DFSClient_NONMAPREDUCE_1610019660_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733192417685/wal.1733192426927 (inode 16678) Holder DFSClient_NONMAPREDUCE_1610019660_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T02:20:38,547 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733192401034/wal.1733192401107 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:38,549 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192428092 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192428092 (inode 16704) Holder DFSClient_NONMAPREDUCE_1610019660_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733192427576/wal.1733192428092 (inode 16704) Holder DFSClient_NONMAPREDUCE_1610019660_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T02:20:38,550 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733192428244/wal.1733192428354 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:20:38,552 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal.1733192400895 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal.1733192400895 (inode 16485) Holder DFSClient_NONMAPREDUCE_1610019660_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733192400707/wal.1733192400895 (inode 16485) Holder DFSClient_NONMAPREDUCE_1610019660_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T02:20:38,554 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal.1733192400477 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal.1733192400477 (inode 16462) Holder DFSClient_NONMAPREDUCE_1610019660_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733192400244/wal.1733192400477 (inode 16462) Holder DFSClient_NONMAPREDUCE_1610019660_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T02:20:38,559 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@211150ab{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T02:20:38,561 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3babed56{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T02:20:38,561 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T02:20:38,561 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70c89de7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T02:20:38,562 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b2b98f9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/hadoop.log.dir/,STOPPED} 2024-12-03T02:20:38,564 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T02:20:38,564 WARN [BP-537057715-172.17.0.3-1733192391930 heartbeating to localhost/127.0.0.1:44855 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T02:20:38,564 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T02:20:38,564 WARN [BP-537057715-172.17.0.3-1733192391930 heartbeating to localhost/127.0.0.1:44855 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-537057715-172.17.0.3-1733192391930 (Datanode Uuid d5b007e8-31d1-4b23-91ab-1d4bdb9ba5b2) service to localhost/127.0.0.1:44855 2024-12-03T02:20:38,565 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data5/current/BP-537057715-172.17.0.3-1733192391930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T02:20:38,565 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data6/current/BP-537057715-172.17.0.3-1733192391930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T02:20:38,566 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T02:20:38,568 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7742c079{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T02:20:38,568 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7dd93ad{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T02:20:38,568 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T02:20:38,568 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ca59792{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T02:20:38,569 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@918952d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/hadoop.log.dir/,STOPPED} 2024-12-03T02:20:38,570 WARN [BP-537057715-172.17.0.3-1733192391930 heartbeating to localhost/127.0.0.1:44855 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T02:20:38,570 WARN [BP-537057715-172.17.0.3-1733192391930 heartbeating to localhost/127.0.0.1:44855 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-537057715-172.17.0.3-1733192391930 (Datanode Uuid 8ba6be36-95ab-4a1f-9b4a-06a3af00259d) service to localhost/127.0.0.1:44855 2024-12-03T02:20:38,570 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T02:20:38,570 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T02:20:38,570 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data3/current/BP-537057715-172.17.0.3-1733192391930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T02:20:38,571 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data4/current/BP-537057715-172.17.0.3-1733192391930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T02:20:38,571 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T02:20:38,572 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2ad3efb4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T02:20:38,573 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3b68aeb4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T02:20:38,573 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T02:20:38,573 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32ec6e08{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T02:20:38,573 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ce178fc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/hadoop.log.dir/,STOPPED} 2024-12-03T02:20:38,574 WARN [BP-537057715-172.17.0.3-1733192391930 heartbeating to localhost/127.0.0.1:44855 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T02:20:38,574 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T02:20:38,574 WARN [BP-537057715-172.17.0.3-1733192391930 heartbeating to localhost/127.0.0.1:44855 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-537057715-172.17.0.3-1733192391930 (Datanode Uuid 05ad8527-09ed-4ccd-9721-cc32ed0ed941) service to localhost/127.0.0.1:44855 2024-12-03T02:20:38,574 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T02:20:38,575 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data1/current/BP-537057715-172.17.0.3-1733192391930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T02:20:38,575 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/cluster_5e621084-ec96-8bab-6c77-66d1ca559e78/data/data2/current/BP-537057715-172.17.0.3-1733192391930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T02:20:38,575 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T02:20:38,580 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@398d2d49{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T02:20:38,581 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T02:20:38,581 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T02:20:38,581 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T02:20:38,581 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb607f7b-f7e7-5e29-8fb4-33651736748b/hadoop.log.dir/,STOPPED} 2024-12-03T02:20:38,592 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T02:20:38,638 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down